docker swarm 集群部署Kafka3.5，彻底告别zookeeper

介绍

1. 本次部署kafka3.5版本，彻底告别zookeeper时代，部署更加轻量，运维更加简单
2. 同时使用比较好用的kafka控制台redpandadata

docker swam 集群搭建

• 详见我的另一篇博客 Docker Swarm集群搭建，不再这里赘述。

docker-compose 文件准备

• docker-compose-kafka3-cluster.yml

version: '3.3'
services:
  kafka-0:
    image: docker.io/bitnami/kafka:3.5
    ports:
      - "9094:9094"
    environment:
      - TZ=Asia/Shanghai
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-0:9092,EXTERNAL://172.25.114.1:9094
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
    volumes:
      - kafka_0_data:/bitnami/kafka
    networks:
      - kafka
    deploy:
      mode: replicated
      replicas: 1
      resources:
        limits: # 资源使用上限
          cpus: "0.50"
          memory: 1G
        reservations: # 随时可以使用的资源
          cpus: "0.25"
          memory: 1G
      placement:
        constraints:
          - node.labels.kafka.replica==1

  kafka-1:
    image: docker.io/bitnami/kafka:3.5
    ports:
      - "9095:9095"
    environment:
      - TZ=Asia/Shanghai
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9095
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-1:9092,EXTERNAL://172.25.114.2:9095
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
    volumes:
      - kafka_1_data:/bitnami/kafka
    networks:
      - kafka
    deploy:
      mode: replicated
      replicas: 1
      resources:
        limits: # 资源使用上限
          cpus: "0.50"
          memory: 1G
        reservations: # 随时可以使用的资源
          cpus: "0.25"
          memory: 1G
      placement:
        constraints:
          - node.labels.kafka.replica==2

  kafka-2:
    image: docker.io/bitnami/kafka:3.5
    ports:
      - "9096:9096"
    environment:
      - TZ=Asia/Shanghai
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9096
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-2:9092,EXTERNAL://172.25.114.3:9096
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
    volumes:
      - kafka_2_data:/bitnami/kafka
    networks:
      - kafka
    deploy:
      mode: replicated
      replicas: 1
      resources:
        limits: # 资源使用上限
          cpus: "0.50"
          memory: 1G
        reservations: # 随时可以使用的资源
          cpus: "0.25"
          memory: 1G
      placement:
        constraints:
          - node.labels.kafka.replica==3

  kafka-console:
    image: docker.io/redpandadata/console:latest
    ports:
      - "9099:8080"
    environment:
      KAFKA_BROKERS: "kafka-0:9092,kafka-1:9092,kafka-2:9092"
    networks:
      - kafka
    depends_on:
      - "kafka-0"
      - "kafka-1"
      - "kafka-2"
    deploy:
      placement:
        constraints:
          - node.labels.kafka.replica==1
volumes:
  kafka_0_data:
    driver: local
  kafka_1_data:
      driver: local
  kafka_2_data:
      driver: local

networks:
  kafka:
    external: true

NOTE: 这个docker-compose文件里面加了资源的限制，生产环境建议取消限制

创建专属网络

docker network create --driver overlay kafka

拉取镜像

docker pull docker.io/bitnami/kafka:3.5
docker pull docker.io/redpandadata/console:latest

给docker node打标签

docker node update --label-add kafka.replica=1 node1
docker node update --label-add kafka.replica=2 node2
docker node update --label-add kafka.replica=3 node3

部署服务

• 部署服务使用 docker stack deploy，其中 -c 参数指定 compose 文件名。

docker stack deploy -c docker-compose-kafka3-cluster.yml