filebeat->kafka->logstash->es->kibana

filebeat部分（k8s）

filebeat.yml

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: kube-system
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      containers:
      - command:
        - /usr/share/filebeat/filebeat
        - -c
        - /etc/filebeat/filebeat.yml
        image: docker.elastic.co/beats/filebeat:7.7.1
        name: filebeat
        volumeMounts:
        - name: logger
          mountPath: /var/log/nginx
        - name: filebeat-config
          mountPath: /etc/filebeat/
      securityContext:
        runAsUser: 0
      volumes:
      - name: logger
        hostPath:
          path: /var/log/nginx
          type: ""
      - name: filebeat-config
        configMap:
          name: filebeat-config

filebeat-config.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /var/log/nginx/*.log
      fields:
        log_topic: test-log
    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
      reload.enabled: false
    setup.template.settings:
      index.number_of_shards: 1
    output.kafka:
      enabled: true
      hosts: ["192.168.1.103:9092"]
      topic: '%{[fields][log_topic]}'
      partition.round_robin:
        reachable_only: true
      worker: 2
      required_acks: 1
      compression: gzip
      max_message_bytes: 10000000
    logging.level: debug

kafka部分（物理机）

kafka3.0.0

wget -c https://archive.apache.org/dist/kafka/3.0.0/kafka_2.13-3.0.0.tgz

zookeeper3.5.9

wget -c wget https://archive.apache.org/dist/zookeeper/zookeeper-3.5.9/apache-zookeeper-3.5.9-bin.tar.gz

logstash部分（k8s）

logstash7.7.1

logstash.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: logstash-elk-test-log
spec:
  replicas: 1
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      labels:
        app: logstash
    spec:
      volumes:
      - name: config
        configMap:
          name: logstash-config
      - name: config-yml
        configMap:
          name: config-yml
      hostname: logstash
      containers:
      - command: 
        - /usr/share/logstash/bin/logstash
        - -f
        - /usr/share/logstash/config/logstash.conf
        name: logstash
        image: docker.elastic.co/logstash/logstash:7.7.1
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: config
          mountPath: "/usr/share/logstash/config/logstash.conf"
          readOnly: true
          subPath: logstash.conf
        - name: config-yml
          mountPath: "/usr/share/logstash/config/logstash.yml"
          readOnly: true
          subPath: logstash.yml

logstash-config.yml创建logstash.conf配置文件

kind: ConfigMap
apiVersion: v1
metadata:
  name: logstash-config
data:
  logstash.conf: |
    input {
        kafka {
            bootstrap_servers => "192.168.1.103:9092"
            group_id => "elk-logstash"
            topics => "test-log"
            codec => "json"
            consumer_threads => 1
            decorate_events => false
            auto_offset_reset => "latest"
            session_timeout_ms => "120000" 
            request_timeout_ms => "240000"
            heartbeat_interval_ms => "10000"
            max_poll_records => "100000"
        }
    }

    filter {
            grok {
                    match => {"message" => "\[%{TIMESTAMP_ISO8601:timestamp}\] \[reqid=%{GREEDYDATA:reqid}\] \[%{GREEDYDATA}"}
            }
            date {
                match => [ "timestamp" , "YYYY-MM-dd HH:mm:ss.SSSSSS" ]
            }
    }

    output {
       elasticsearch {
                            hosts => ["192.168.1.103:9200"]
                            index => "logstash-test-log-%{+YYYY.MM.dd}"
       }
    }

logstash-config2创建logstash.yml文件

kind: ConfigMap
apiVersion: v1
metadata:
  name: config-yml
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.hosts: [ "http://192.168.1.103:9200" ]

elasticsearch部分（物理机）

elasticsearch7.7.1

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.7.1-linux-x86_64.tar.gz

kibana部分（物理机）

kibana7.7.1

[root@node4 ~]# cat /etc/yum.repos.d/kibana.repo 
[kibana-7.x]
name=Kibana repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md