filebeat收集k8s日志

配置文件介绍

filebeat.inputs:
- type: log
  #开启filebeat采集软连接的文件
  symlinks: true
  #给当前的输入类型打上标签
  tags: ["kube-system"]
  #自定义字段
  fields:
    namespace: "kube-system"
  #是否将自定义字段设置为一级模式
  fields_under_root: true
  paths:
    - /root/kube-system_*/*/*.log

output.elasticsearch:
  hosts: ["172.17.68.100:9200"]
  indices:
    - index: "local-kube-system-%{+yyyy.MM.dd}"
      when.contains:
        namespace: "kube-system"
    - index: "local-default-%{+yyyy.MM.dd}"
      when.contains:
        namespace: "default"

#关闭索引的生命周期，开启则上面的index配置会被无视
setup.ilm.enabled: false
#设置索引模板的名称
setup.template.nameo: "local"
#设置索引模板的匹配模式
setup.template.pattern: "local-*"
#覆盖已有的索引模板
setup.template.overwrite: false
#设置索引分片与副本数量
setup.template.settings:
  index.number_of_shards: 1
  index.number_of_replicas: 0

官网下载对应的helm-filebeat的chart版本 https://github.com/elastic/helm-charts
修改values.yaml的值，使得helm部署后，configmap的内容如下（）下面测试了kube-system和test空间

filebeat.inputs:
- type: container
  paths:
    - /var/log/containers/*.log
  processors:
    - add_kubernetes_metadata:
        default_indexers.enabled: true
        default_matchers.enabled: true
        host: ${NODE_NAME}
        matchers:
        - logs_path:
            logs_path: "/var/log/containers/"
    - drop_event.when.regexp:
        or:
          kubernetes.pod.name: "filebeat-*"
          kubernetes.pod.name: "external-dns.*"
          kubernetes.pod.name: "coredns-*"
    - drop_fields:
        fields: 
          - log
          - input
          - container.*
          - kubernetes.labels
          - kubernetes.node
          - kubernetes.pod.id
output.elasticsearch:
  host: '${NODE_NAME}'
  hosts: ["172.17.68.100:9200"]
  indices:
    - index: "efk-test-kube-system-%{+yyyy.MM.dd}"
      when.contains:
        kubernetes.namespace: "kube-system"

    - index: "efk-test-test-%{+yyyy.MM.dd}"
      when.contains:
        kubernetes.namespace: "test"
#关闭索引的生命周期，开启则上面的index配置会被无视
setup.ilm.enabled: false
#设置索引模板的名称
setup.template.nameo: "efk-test"
#设置索引模板的匹配模式
setup.template.pattern: "efk-test-*"
#覆盖已有的索引模板
setup.template.overwrite: false
#设置索引分片与副本数量
setup.template.settings:
  index.number_of_shards: 1
  index.number_of_replicas: 0