docker部署Zookeeper+Storm+Kafka
安装docker
信任Docker的GPG公钥:
curl -fsSL https://repo.huaweicloud.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -`
添加软件仓库:
sudo add-apt-repository "deb [arch=amd64] https://repo.huaweicloud.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
更新索引文件并安装
sudo apt-get update
sudo apt-get install docker-ce
创建专用网段
docker network create --subnet 172.30.0.0/24 --gateway 172.30.0.1 zookeeper
创建Zookeeper Compose文件
version: '3'
services:
zookeeper1:
image: zookeeper:3.4
restart: always
hostname: zookeeper1
container_name: zookeeper1
ports:
- "2181:2181"
volumes:
- "/home/zk/workspace/volumes/zkcluster/zookeeper1/data:/data"
- "/home/zk/workspace/volumes/zkcluster/zookeeper1/datalog:/datalog"
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zookeeper2:2888:3888 server.3=zookeeper3:2888:3888
networks:
zookeeper:
ipv4_address: 172.30.0.11
zookeeper2:
image: zookeeper:3.4
restart: always
hostname: zookeeper2
container_name: zookeeper2
ports:
- "2182:2181"
volumes:
- "/home/zk/workspace/volumes/zkcluster/zookeeper2/data:/data"
- "/home/zk/workspace/volumes/zkcluster/zookeeper2/datalog:/datalog"
environment:
ZOO_MY_ID: 2
ZOO_SERVERS: server.1=zookeeper1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zookeeper3:2888:3888
networks:
zookeeper:
ipv4_address: 172.30.0.12
zookeeper3:
image: zookeeper:3.4
restart: always
hostname: zookeeper3
container_name: zookeeper3
ports:
- "2183:2181"
volumes:
- "/home/zk/workspace/volumes/zkcluster/zookeeper3/data:/data"
- "/home/zk/workspace/volumes/zkcluster/zookeeper3/datalog:/datalog"
environment:
ZOO_MY_ID: 3
ZOO_SERVERS: server.1=zookeeper1:2888:3888 server.2=zookeeper2:2888:3888 server.3=0.0.0.0:2888:3888
networks:
zookeeper:
ipv4_address: 172.30.0.13
networks:
zookeeper:
external:
name: zookeeper
创建Kafka Compose文件
version: '3'
services:
kafka1:
image: wurstmeister/kafka:2.12-2.4.1
restart: always
hostname: kafka1
container_name: kafka1
privileged: true
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka1
KAFKA_LISTENERS: PLAINTEXT://kafka1:9092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
volumes:
- /home/zk/workspace/volumes/kafkaCluster/kafka1/logs:/kafka
- /home/w2110276138/dataset/:/tmp
networks:
kafka:
ipv4_address: 172.30.0.14
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
kafka2:
image: wurstmeister/kafka:2.12-2.4.1
restart: always
hostname: kafka2
container_name: kafka2
privileged: true
ports:
- 9093:9093
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka2
KAFKA_LISTENERS: PLAINTEXT://kafka2:9093
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9093
KAFKA_ADVERTISED_PORT: 9093
KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
volumes:
- /home/zk/workspace/volumes/kafkaCluster/kafka2/logs:/kafka
- /home/w2110276138/dataset/:/tmp
networks:
kafka:
ipv4_address: 172.30.0.15
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
kafka3:
image: wurstmeister/kafka:2.12-2.4.1
restart: always
hostname: kafka3
container_name: kafka3
privileged: true
ports:
- 9094:9094
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka3
KAFKA_LISTENERS: PLAINTEXT://kafka3:9094
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9094
KAFKA_ADVERTISED_PORT: 9094
KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
volumes:
- /home/zk/workspace/volumes/kafkaCluster/kafka3/logs:/kafka
- /home/w2110276138/dataset/:/tmp
networks:
kafka:
ipv4_address: 172.30.0.16
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
networks:
kafka:
external:
name: zookeeper
创建Storm Compose文件
version: '3'
services:
nimbus:
image: storm:1.2.2
command: storm nimbus
restart: always
environment:
- STORM_ZOOKEEPER_SERVERS=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
container_name: nimbus
hostname: nimbus
networks:
storm:
ipv4_address: 172.30.0.17
ports:
- 6627:6627
volumes:
- "./storm.yaml:/conf/storm.yaml"
- "/home/w2110276138/jar:/tmp"
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
kafka1: 172.30.0.14
kafka2: 172.30.0.15
kafka3: 172.30.0.16
ui: 172.30.0.18
ui:
image: storm:1.2.2
command: storm ui
restart: always
environment:
- STORM_ZOOKEEPER_SERVERS=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
container_name: ui
hostname: ui
networks:
storm:
ipv4_address: 172.30.0.18
ports:
- 8080:8080
volumes:
- "./storm.yaml:/conf/storm.yaml"
depends_on:
- nimbus
links:
- nimbus:nimbus
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
kafka1: 172.30.0.14
kafka2: 172.30.0.15
kafka3: 172.30.0.16
nimbus: 172.30.0.17
supervisor:
image: storm:1.2.2
command: storm supervisor
restart: always
environment:
- STORM_ZOOKEEPER_SERVERS=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
networks:
- storm
depends_on:
- nimbus
links:
- nimbus:nimbus
volumes:
- "./storm.yaml:/conf/storm.yaml"
extra_hosts:
zookeeper1: 172.30.0.11
zookeeper2: 172.30.0.12
zookeeper3: 172.30.0.13
kafka1: 172.30.0.14
kafka2: 172.30.0.15
kafka3: 172.30.0.16
networks:
storm:
external:
name: zookeeper
启动Zookeeper+Kafka+Storm
如果把所有的compose文件写在一起,则可以一起启动。
docker compose -f docker-compose-zookeeper.yml up -d
docker compose -f docker-compose-kafka.yml up -d
docker compose -f docker-compose-storm.yml up -d
扩展Supervisor
docker compose -f docker-compose-storm.yml up --scale supervisor=4 -d
标签:storm,Zookeeper,2181,Storm,docker,172.30,zookeeper1,zookeeper2,zookeeper3
From: https://www.cnblogs.com/Modest-Hamilton/p/17515034.html