logstash conf example

# this is for logstash configuration
input {
  beats {
    port => 5044
  }
}

filter {
   grok {
   match => {
      "message" => "%{TIMESTAMP_ISO8601:timestamp} \[%{DATA:log_level}\] %{DATA:url} (?<content>(.)*)"
    }
   }

  if "_grokparsefailure" not in [tags] {
    if [content] {
      json {
        source => "content"
      }

      if [dest] == "mongo" {
        mutate {
          add_field => {"logType" => "query"}
        }
      }
      else {
        mutate {
          add_field => {"logType" => "log"}
        }
      }

    }

    mutate {
      remove_field => [ "content", "message", "tags"]
      gsub => ["timestamp", ",[0-9]+", ""]
    }
  }
  else {
    mutate {
      add_field => {"logType" => "else"}
    }
  }
}

output {
  if [logType] == "query" { # this is for query log
    mongodb {
      collection => "log"
      database => "gamedb"
      uri => "mongodb://"
      codec => json
      isodate => true
    }
  }
  else if [logType] == "log" and [log_level] == "INFO" {
    s3 {
      access_key_id => ""
      secret_access_key => ""
      region => "us-east-1"
      prefix => "%{+YYYY/MM/dd}/"
      bucket => ""
      size_file => 100000  # size: bytes,  100000 = 100kbytes
      time_file => 5
      codec => "json_lines"
    }
  }
  elasticsearch {
      hosts => [search.~~.es.aws.com:80]
  }
}