ELK（Elasticsearch、Kibana、Filebeat、Metricbeat、Logstash、Elastic Agent、Fleet Server、APM）备忘录

（241018）

.env

# 项目名
COMPOSE_PROJECT_NAME=es
# elastic用户（至少6个数字）
ELASTIC_PASSWORD=
# kibana用户（至少6个数字）
KIBANA_PASSWORD=
# 版本号，一定要填写需求版本
# https://www.elastic.co/downloads/past-releases#elasticsearch
STACK_VERSION=8.15.1
# 集群名
CLUSTER_NAME=docker-cluster
# elastic的使用版本
LICENSE=basic
# elastic端口
ES_PORT=9200
# kibana端口
KIBANA_PORT=5601
# fleet端口
FLEET_PORT=8220
# apm端口
APMSERVER_PORT=8200
# APM令牌，k8s环境使用Kubernetes Secrets管理，也可以linux环境配置
ELASTIC_APM_SECRET_TOKEN=
# es、kibana、logstash最大内存量，可以换写成byte（3073741824、1073741824）
ES_MEM_LIMIT=3g
KB_MEM_LIMIT=1g
LS_MEM_LIMIT=1g

# 密钥，k8s环境使用Kubernetes Secrets管理，也可以linux环境配置
ENCRYPTION_KEY=

docker-compose.yml

x${ELASTIC_PASSWORD} == x和x${KIBANA_PASSWORD} == x检测变量是否存在
! -f config/certs/ca.zip和! -f config/certs/certs.zip检测证书是否存在
bootstrap.memory_lock启用内存锁定，防止交换
xpack.license.self_generated.type指定许可证类型
（端口只适用于本地开发，生产环境更改端口）

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local
  metricbeatdata01:
    driver: local
  filebeatdata01:
    driver: local
  logstashdata01:
    driver: local
  fleetserverdata:
    driver: local

networks:
  default:
    name: es-network
    external: false

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: kibana\n"\
          "    dns:\n"\
          "      - kibana\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "Done";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    labels:
      co.elastic.logs/module: elasticsearch
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - discovery.type=single-node
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${ES_MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    labels:
      co.elastic.logs/module: kibana
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
      - ./kibana.yml:/usr/share/kibana/config/kibana.yml:ro
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
      - XPACK_REPORTING_KIBANASERVER_HOSTNAME=localhost
      - SERVER_SSL_ENABLED=true
      - SERVER_SSL_CERTIFICATE=config/certs/kibana/kibana.crt
      - SERVER_SSL_KEY=config/certs/kibana/kibana.key
      - SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
      - ELASTIC_APM_SECRET_TOKEN=${ELASTIC_APM_SECRET_TOKEN}
    mem_limit: ${KB_MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  metricbeat01:
    depends_on:
      es01:
        condition: service_healthy
      kibana:
        condition: service_healthy
    image: docker.elastic.co/beats/metricbeat:${STACK_VERSION}
    user: root
    volumes:
      - certs:/usr/share/metricbeat/certs
      - metricbeatdata01:/usr/share/metricbeat/data
      - "./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro"
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
      - "/proc:/hostfs/proc:ro"
      - "/:/hostfs:ro"
    environment:
      - ELASTIC_USER=elastic
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTIC_HOSTS=https://es01:9200
      - KIBANA_HOSTS=https://kibana:5601
      - LOGSTASH_HOSTS=http://logstash01:9600
      - CA_CERT=certs/ca/ca.crt
      - ES_CERT=certs/es01/es01.crt
      - ES_KEY=certs/es01/es01.key
      - KB_CERT=certs/kibana/kibana.crt
      - KB_KEY=certs/kibana/kibana.key
    command:
      -strict.perms=false

  filebeat01:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
    user: root
    volumes:
      - certs:/usr/share/filebeat/certs
      - filebeatdata01:/usr/share/filebeat/data
      - "./filebeat_ingest_data/:/usr/share/filebeat/ingest_data/"
      - "./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro"
      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    environment:
      - ELASTIC_USER=elastic
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTIC_HOSTS=https://es01:9200
      - KIBANA_HOSTS=https://kibana:5601
      - LOGSTASH_HOSTS=http://logstash01:9600
      - CA_CERT=certs/ca/ca.crt
    command:
      -strict.perms=false

  logstash01:
    depends_on:
      es01:
        condition: service_healthy
      kibana:
        condition: service_healthy
    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
    labels:
      co.elastic.logs/module: logstash
    user: root
    volumes:
      - certs:/usr/share/logstash/certs
      - logstashdata01:/usr/share/logstash/data
      - "./logstash_ingest_data/:/usr/share/logstash/ingest_data/"
      - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"
    environment:
      - xpack.monitoring.enabled=false
      - ELASTIC_USER=elastic
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTIC_HOSTS=https://es01:9200

  fleet-server:
    depends_on:
      kibana:
        condition: service_healthy
      es01:
        condition: service_healthy
    image: docker.elastic.co/beats/elastic-agent:${STACK_VERSION}
    volumes:
      - certs:/certs
      - fleetserverdata:/usr/share/elastic-agent
      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
      - "/proc:/hostfs/proc:ro"
      - "/:/hostfs:ro"
    ports:
      - ${FLEET_PORT}:8220
      - ${APMSERVER_PORT}:8200
    user: root
    environment:
      - SSL_CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt
      - CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt
      - FLEET_CA=/certs/ca/ca.crt
      - FLEET_ENROLL=1
      - FLEET_INSECURE=true
      - FLEET_SERVER_ELASTICSEARCH_CA=/certs/ca/ca.crt
      - FLEET_SERVER_ELASTICSEARCH_HOST=https://es01:9200
      - FLEET_SERVER_ELASTICSEARCH_INSECURE=true
      - FLEET_SERVER_ENABLE=1
      - FLEET_SERVER_CERT=/certs/fleet-server/fleet-server.crt
      - FLEET_SERVER_CERT_KEY=/certs/fleet-server/fleet-server.key
      - FLEET_SERVER_INSECURE_HTTP=true
      - FLEET_SERVER_POLICY_ID=fleet-server-policy
      - FLEET_URL=https://fleet-server:8220
      - KIBANA_FLEET_CA=/certs/ca/ca.crt
      - KIBANA_FLEET_SETUP=1
      - KIBANA_FLEET_USERNAME=elastic
      - KIBANA_FLEET_PASSWORD=${ELASTIC_PASSWORD}
      - KIBANA_HOST=https://kibana:5601

  webapp:
    build:
      context: app
    volumes:
      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
      - "/proc:/hostfs/proc:ro"
      - "/:/hostfs:ro"
    ports:
      - 8000:8000

metricbeat.yml

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

metricbeat.modules:
- module: elasticsearch
  xpack.enabled: true
  period: 10s
  hosts: ${ELASTIC_HOSTS}
  username: ${ELASTIC_USER}
  password: ${ELASTIC_PASSWORD}
  ssl:
    enabled: true
    certificate_authorities: ${CA_CERT}

- module: logstash
  xpack.enabled: true
  period: 10s
  hosts: ${LOGSTASH_HOSTS}

- module: kibana
  metricsets: 
    - stats
  period: 10s
  hosts: ${KIBANA_HOSTS}
  username: ${ELASTIC_USER}
  password: ${ELASTIC_PASSWORD}
  xpack.enabled: true
  ssl:
    enabled: true
    certificate_authorities: ${CA_CERT}

- module: docker
  metricsets:
    - "container"
    - "cpu"
    - "diskio"
    - "healthcheck"
    - "info"
    #- "image"
    - "memory"
    - "network"
  hosts: ["unix:///var/run/docker.sock"]
  period: 10s
  enabled: true

processors:
  - add_host_metadata: ~
  - add_docker_metadata: ~

output.elasticsearch:
  hosts: ${ELASTIC_HOSTS}
  username: ${ELASTIC_USER}
  password: ${ELASTIC_PASSWORD}
  ssl:
    enabled: true
    certificate_authorities: ${CA_CERT}

filebeat.yml

filebeat.inputs:
- type: filestream
  id: default-filestream
  paths:
    - ingest_data/*.log

filebeat.autodiscover:
  providers:
    - type: docker
      hints.enabled: true

processors:
- add_docker_metadata: ~

setup.kibana:
  host: ${KIBANA_HOSTS}
  username: ${ELASTIC_USER}
  password: ${ELASTIC_PASSWORD}

output.elasticsearch:
  hosts: ${ELASTIC_HOSTS}
  username: ${ELASTIC_USER}
  password: ${ELASTIC_PASSWORD}
  ssl:
    enabled: true
    certificate_authorities: ${CA_CERT}

logstash.conf

input {
  file {
    #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html
    #default is TAIL which assumes more data will come into the file.
    #change to mode => "read" if the file is a compelte file.  by default, the file will be removed once reading is complete -- backup your files if you need them.
    mode => "tail"
    path => "/usr/share/logstash/ingest_data/*"
  }
}

filter {
}

output {
  elasticsearch {
    index => "logstash-%{+YYYY.MM.dd}"
    hosts=> "${ELASTIC_HOSTS}"
    user=> "${ELASTIC_USER}"
    password=> "${ELASTIC_PASSWORD}"
    cacert=> "certs/ca/ca.crt"
  }
}

kibana.yml

elastic:
  apm:
    active: true
    serverUrl: "http://fleet-server:8200"
    secretToken: ${ELASTIC_APM_SECRET_TOKEN}
server.host: "0.0.0.0"
telemetry.enabled: "true"
xpack.fleet.packages:
  - name: fleet_server
    version: latest
  - name: system
    version: latest
  - name: elastic_agent
    version: latest
  - name: apm
    version: latest
xpack.fleet.agentPolicies:
  - name: Fleet-Server-Policy
    id: fleet-server-policy
    namespace: default
    monitoring_enabled: 
      - logs
      - metrics
    package_policies:
      - name: fleet_server-1
        package:
          name: fleet_server
      - name: system-1
        package:
          name: system
      - name: elastic_agent-1
        package:
          name: elastic_agent
      - name: apm-1
        package:
          name: apm
        inputs:
        - type: apm
          enabled: true
          vars:
          - name: host
            value: 0.0.0.0:8200
          - name: secret_token
            value: ${ELASTIC_APM_SECRET_TOKEN}

app/dockerfile（仅演示文件）

# syntax=docker/dockerfile:1

FROM python:3.9-slim-buster

WORKDIR /app

COPY requirements.txt requirements.txt

RUN pip3 install -r requirements.txt

COPY main.py main.py

CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "info", "--workers", "1"]

app/main.py（仅演示文件）

from elasticapm.contrib.starlette import ElasticAPM, make_apm_client
from fastapi import FastAPI
from nicegui import ui
from typing import Callable
import asyncio
import functools
import httpx as r
#import psutil

try:
  apm = make_apm_client({
      'SERVICE_NAME': 'my_python_service',
      'SECRET_TOKEN': 'supersecrettoken',
      # SERVER_URL must be set to "fleet-server" if running as a docker container.
      # if running as a local python script, then set the url to "LOCALHOST"
      'SERVER_URL': 'http://fleet-server:8200',
      'ENVIRONMENT': 'development'
  })
except Exception as e:
  print('failed to create client')

app = FastAPI()

try:
  app.add_middleware(ElasticAPM, client=apm)
except Exception as e:
  print('failed to add APM Middleware')


@app.get("/custom_message/{message}")
async def custom_message(message: str):
    apm.capture_message(f"Custom Message: {message}")
    return {"message": f"Custom Message:  {message}"}


@app.get("/error")
async def throw_error():
    try:
        1 / 0
    except Exception as e:
        apm.capture_exception()
    return {"message": "Failed Successfully :)"}


def init(fastapi_app: FastAPI) -> None:
    @ui.page('/', title="APM Demo App")
    async def show():
        with ui.header(elevated=True).style('background-color: #3874c8').classes('items-center justify-between'):
            ui.markdown('### APM DEMO')
            ui.button(on_click=lambda: right_drawer.toggle(), icon='menu').props('flat color=white')
        with ui.right_drawer(fixed=False).style('background-color: #ebf1fa').props('bordered') as right_drawer:
            ui.chat_message('Hello Elastic Stack User!',
                            name='APM Robot',
                            stamp='now',
                            avatar='https://robohash.org/apm_robot')
            ui.chat_message('This app is powered by NICEGUI and FastAPI with Elastic APM Instrumentation :)',
                            name='APM Robot',
                            stamp='now',
                            avatar='https://robohash.org/apm_robot')
            ui.chat_message('Please click a button to trigger an APM event.',
                            name='APM Robot',
                            stamp='now',
                            avatar='https://robohash.org/apm_robot')
        with ui.footer().style('background-color: #3874c8'):
            ui.label('APM DEMO PAGE')

        with ui.card():
            ui.label('Generate Error - Python')
            ui.button('Generate', on_click=python_error)

        with ui.card():
            ui.label('Generate Error - JS')
            ui.button('Generate', on_click=js_error)

        with ui.card():
            ui.label('Generate Custom Message')
            custom_message_text = ui.input(placeholder='Message')
            ui.button('Generate').on('click', handler=lambda: gen_custom_message(custom_message_text.value))

    ui.run_with(
        fastapi_app,
        storage_secret='supersecret',  # NOTE setting a secret is optional but allows for persistent storage per user
    )


async def io_bound(callback: Callable, *args: any, **kwargs: any):
    '''Makes a blocking function awaitable; pass function as first parameter and its arguments as the rest'''
    return await asyncio.get_event_loop().run_in_executor(None, functools.partial(callback, *args, **kwargs))


async def python_error():
    try:
        res = await io_bound(r.get, 'http://localhost:8000/error')
        ui.notify(res.text)
    except Exception as e:
        apm.capture_exception()
        ui.notify(f'{e}')


async def js_error():
    try:
        res = await ui.run_javascript('fetch("http://localhost:8000/error")')
        ui.notify(f'Message: Failed Successfully :)')
    except Exception as e:
        apm.capture_exception()
        ui.notify(f'{e}')


async def gen_custom_message(text_message):
    try:
        res = await io_bound(r.get, 'http://localhost:8000/custom_message/' + str(text_message))
        ui.notify(res.text)
    except Exception as e:
        apm.capture_exception()
        ui.notify(f'{e}')

init(app)

try:
  apm.capture_message('App Loaded, Hello World!')
except Exception as e:
  print('error: ' + e)

if __name__ == '__main__':
    print('Please start the app with the "uvicorn" command as shown in the start.sh script')

golang版本（仅演示文件）

# syntax=docker/dockerfile:1

FROM golang:1.20 AS builder

WORKDIR /app

COPY go.mod go.sum ./

RUN go mod download

COPY . .

RUN go build -o main .

FROM gcr.io/distroless/base

WORKDIR /app

COPY --from=builder /app/main .

EXPOSE 8000

CMD ["./main"]

package main

import (
	"fmt"
	"net/http"

	"github.com/gin-gonic/gin"
	"github.com/elastic/apm-agent-go/v2/apm"
	"github.com/elastic/apm-agent-go/v2/module/apmhttp"
)

func main() {
	// Initialize the Elastic APM agent
	tracer, err := apm.NewTracer("my_go_service", "development")
	if err != nil {
		fmt.Println("Failed to create APM tracer:", err)
		return
	}
	defer tracer.Close()

	// Create a Gin router
	r := gin.Default()

	// Middleware to capture APM transactions
	r.Use(func(c *gin.Context) {
		tx := tracer.StartTransaction(c.Request.URL.Path, "request")
		defer tx.End()

		c.Next()

		// Capture errors if any
		if len(c.Errors) > 0 {
			for _, err := range c.Errors {
				apm.CaptureError(err.Err).Send()
			}
		}
	})

	// Custom message endpoint
	r.GET("/custom_message/:message", func(c *gin.Context) {
		message := c.Param("message")
		apm.CaptureMessage(fmt.Sprintf("Custom Message: %s", message)).Send()
		c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("Custom Message: %s", message)})
	})

	// Error endpoint
	r.GET("/error", func(c *gin.Context) {
		defer func() {
			if r := recover(); r != nil {
				apm.CaptureError(fmt.Errorf("recovered from panic: %v", r)).Send()
				c.JSON(http.StatusInternalServerError, gin.H{"message": "Failed Successfully :)"})
			}
		}()
		// This will cause a panic
		_ = 1 / 0
	})

	// Start the server
	if err := r.Run(":8000"); err != nil {
		fmt.Println("Failed to start server:", err)
	}
}

get-cert-details.sh

docker cp es-es01-1:/usr/share/elasticsearch/config/certs/ca/ca.crt /tmp/.
cat /tmp/ca.crt
echo 'ssl:';
echo '  certificate_authorities:';
echo '  - |';
cat /tmp/ca.crt | sed 's/^/    /'
openssl x509 -fingerprint -sha256 -noout -in /tmp/ca.crt | awk -F"=" {' print $2 '} | sed s/://g