logstast filebeat 文件

logstast   filebeat  文件

#ample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}


#output {
#       stdout { codec => rubydebug }
#}

filter {
     json {
         source => "message"
         skip_on_invalid_json => "true"
}
     date {
          match => ["time", "yyyy-MM-dd HH:mm:ss.SSS"]
          target => "@timestamp"
}

geoip {
        source => "client"
        fields => ["city_name", "country_code2", "country_name", "latitude", "longitude", "region_name"]
        remove_field => ["[geoip][latitude]", "[geoip][longitude]"]
    }
    mutate {
        split => [ "upstreamtime", "," ]
    }
    mutate {
        convert => [ "upstreamtime", "float" ]
    }


 useragent {
         source => "http_user_agent"
         target => "os_agent"
    }

}

output {
#nginx
       if [fields][log_type] == "nginx-log-201-83" {
         elasticsearch {
            hosts => ["10.10.201.80:9200"]
            index => "nginx-log-201-83-%{+yyyy.MM.dd}"
        }
       }
       if [fields][log_type] == "nginx-log-201-28" {
         elasticsearch {
           hosts => ["10.10.201.80:9200"]
            index => "nginx-log-201-28-%{+yyyy.MM.dd}"
        }
       }
       if [fields][log_type] == "nginx-log-201-29" {
         elasticsearch {
             hosts => ["10.10.201.80:9200"]
            index => "nginx-log-201-29-%{+yyyy.MM.dd}"
        }
       }
       if [fields][log_type] == "nginx-log-204-11" {
         elasticsearch {
             hosts => ["10.10.201.80:9200"]
            index => "nginx-log-204-11-%{+yyyy.MM.dd}"
        }
       }
       if [fields][log_type] == "nginx-log-201-158" {
         elasticsearch {
             hosts => ["10.10.201.80:9200"]
            index => "nginx-log-201-158-%{+yyyy.MM.dd}"
        }
       }

#java
      if [fields][log_type] == "java-log-201-83" {
         elasticsearch {
            hosts => ["10.10.201.80:9200"]
            index => "java-log-201-83-%{+yyyy.MM.dd}"
        }
       }

#es
      if [fields][log_type] == "es-log-201-80" {
         elasticsearch {
            hosts => ["10.10.201.80:9200"]
            index => "es-log-201-80-%{+yyyy.MM.dd}"
        }
       }


}

filebeat

# Below are the input specific configurations.

- type: log

  # Change to true to enable this input configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /var/log/nginx/*.log
    #- c:\programdata\elasticsearch\logs\*
  fields:
    log_type: "nginx-log-204-11"
  # Exclude lines. A list of regular expressions to match. It drops the lines that are
  # matching any regular expression from the list.
  #exclude_lines: ['^DBG']
  
  
  
  
  
  #----------------------------- Logstash output --------------------------------
#output.logstash:
  # The Logstash hosts
  #hosts: ["localhost:5044"]
output.logstash:
   # The Logstash hosts
      hosts: ["10.10.201.81:5044"]
  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"