k8s通过sidecar模式收集pod的容器日志至ELK

架构：

已完成的部署

1、ES集群及kibana部署

​​https://blog.51cto.com/yht1990/6080981​​

2、kafaka+zookeeper集群

​​https://blog.51cto.com/yht1990/6081518​​

准备sidecar镜像(filebeat)

找一台服务器打镜像

[root@yw-test filebeat]# cat Dockerfile 
FROM docker.elastic.co/beats/filebeat:7.9.0
COPY filebeat.yml /usr/share/filebeat/filebeat.yml
USER root
RUN chown root:filebeat /usr/share/filebeat/filebeat.yml
USER filebeat
[root@yw-test filebeat]# 
[root@yw-test filebeat]# 
[root@yw-test filebeat]# cat filebeat.yml 
filebeat.inputs:
- input_type: log
  paths:
    - /data/logic/log/*.log
  fields:
    log_topic: "${TOPIC_ID}"
  #tail_files: true
  clean_inactive: 48h
  ignore_older: 24h
  close_inactive: 1m

output.kafka:
  hosts: "${KAFKA_SERVER}"
  topic: '%{[fields.log_topic]}'
  partition.round_robin:
    reachable_only: true
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

logging.level: error

[root@yw-test filebeat]# docker build . -t 10.0.7.12/k8s/filebeat/sidecar:7.9.0
[root@yw-test filebeat]# docker push 10.0.7.12/k8s/filebeat/sidecar:7.9.0

创建deployment

[root@k8s-test-manager src]# cat test-clavaplus-consume-sidecar-filebeat.yml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: test-clavaplus-consume-deployment-label
  name: test-clavaplus-consume-deployment
  namespace: test-app
spec:
  replicas: 2
  selector:
    matchLabels:
      app: test-clavaplus-consume-selector
  template:
    metadata:
      labels:
        app: test-clavaplus-consume-selector
    spec:
      imagePullSecrets:
      - name: myregistrykey
      containers:
      - name: test-clavaplus-consume
        image: registry.cn-shenzhen.aliyuncs.com/test/swoole:test-clavaplus-consume-518b279-2023-0227-1833
        imagePullPolicy: IfNotPresent
        #imagePullPolicy: Always
        ports:
        - containerPort: 9501
          protocol: TCP
          name: http
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 200m
            memory: "512Mi"
        volumeMounts:
        - name: applogs
          mountPath: /data/logic/log
      - name: sidecar-filebeat
        image: 10.0.7.12/k8s/filebeat/sidecar:7.9.0
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        env:
        - name: "TOPIC_ID"
          value: "test-clavaplus-consume"
        - name: "KAFKA_SERVER"
          value: '["10.0.7.53:9092", "10.0.7.54:9092", "10.0.7.55:9092"]'
        volumeMounts:
        - name: applogs
          mountPath: /data/logic/log
      volumes:
      - name: applogs #定义通过emptyDir实现业务容器与sidecar容器的日志共享，以让sidecar收集业务容器中的日志
        emptyDir: {}

创建deployement

kubectl apply -f test-clavaplus-consume-sidecar-filebeat.yml

kafka工具检查有无数据

可以看到目前已有数据

部署logstash

yum -y install java-1.8.0-openjdk
yum -y install https://mirror.tuna.tsinghua.edu.cn/elasticstack/7.x/yum/7.9.0/logstash-7.9.0.rpm
systemctl start logstash
systemctl enable logstash
tail -f /var/log/logstash/logstash-plain.log

创建logstash配置文件

# 将收集过来的json日志做json解析

    codec => json {

       charset => "UTF-8" # 支持中文搜索

   }

# 将message消息赋值给source字段

filter {

 json {

   source => "message"

 }

 mutate {    

  # 删除没用的字段

   remove_field => ["message","@version","agent","ecs"]

 }

}

input {
  kafka {
    bootstrap_servers => "10.0.7.53:9092,10.0.7.54:9092,10.0.7.55:9092"
    topics => ["test-clavaplus-consume"] 
    codec => json {
        charset => "UTF-8"
    }
  }
}

filter {
  json {
    source => "message"
  }
 if [fields][log_topic] == "test-clavaplus-consume"  {
    mutate {
     remove_field => ["message","@version","agent","ecs"]
    }
  }
}

output {
  if [fields][log_topic] == "test-clavaplus-consume"  {
    elasticsearch {
      hosts => ["10.0.7.46:9200","10.0.7.47:9200","10.0.7.48:9200"]
      index => "test-clavaplus-consume-%{+YYYY.MM.dd}"
    }
  }
}

启动logstash

systemctl restart logstash && tail -f /var/log/logstash/logstash-plain.log

登陆kibana创建索引并查看数据