kafka集群部署教程

#### 部署zookeeper集群

1、先部署zookeeper集群一般是3个节点,安装zookeeper 依赖java 环境先安装 jdk

      yum install java-1.8.0-openjdk-devel -y

      java -version #查看jdk版本

2、创建工作目录（3个节点都需要执行）

    mkdir -p /usr/local/zookeeper

    日志存储目录（3个节点都需要执行）

   mkdir -p /usr/local/zookeeper/dataDir

3、下载安装包并解压(3个节点都需要执行)

     cd /usr/local/zookeeper

     wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.8.0/apache-zookeeper-3.8.0-bin.tar.gz

     tar -xvf  apache-zookeeper-3.8.0-bin.tar.gz  

4、新增实例文件（3个节点都需要执行）myid文件后续zoo.cfg中使用

echo "1" > /usr/local/zookeeper/dataDir/myid

echo "2" > /usr/local/zookeeper/dataDir/myid

echo "3" > /usr/local/zookeeper/dataDir/myid

5、配置修改,并scp 到另外2台机器上面

cd /usr/local/zookeeper/apache-zookeeper-3.8.0-bin/conf

mv zoo_sample.cfg zoo.cfg

vim  zoo.cfg

egrep -v "^#|^$" zoo.cfg

tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper/dataDir
server.1=10.x.x.2:2888:3888
server.2=10.x.x.3:2888:3888
server.3=10.x.x.4:2888:3888
clientPort=2181
maxClientCnxns=60

6、关闭防火墙

systemctl stop firewalld && systemctl disable firewalld

7、配置systemd 服务

cd /usr/lib/systemd/system/

创建zookeeper 服务文件

cat zookeeper.service

[Unit]
Description=zookeeper.service
After=network.target

[Service]
Type=forking
Environment="PATH=/usr/local/jdk1.8.0_311/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"
User=root
Group=root
ExecStart=/usr/local/zookeeper/apache-zookeeper-3.8.0-bin/bin/zkServer.sh start
ExecStop=/usr/local/zookeeper/apache-zookeeper-3.8.0-bin/bin/zkServer.sh stop
Restart=on-failure

[Install]
WantedBy=multi-user.target

刷新 systemctl 

systemctl daemon-reload

 systemctl enable  --now zookeeper

8、 查看集群状态

/usr/local/zookeeper/apache-zookeeper-3.8.0-bin/bin/zkServer.sh status

#######部署kafka集群###########

1、创建工作目录（3个节点都需要执行）

mkdir -p /usr/local/kafka

2、下载安装包解压(3个节点都需要执行)

cd /usr/local/kafka

wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/3.4.0/kafka_2.13-3.4.0.tgz

tar zxf  kafka_2.13-3.4.0.tgz

3、cd /usr/local/kafka/kafka_2.13-3.4.0/config （3个节点都需要只需要关注修改项broker.id、listeners、zookeeper.connect、log.dirs）

#节点1#

vim server.properties

broker.id=0
listeners=PLAINTEXT://10.x.x.2:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/var/log/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.x.x.2:2181,10.x.x.3:2181,10.x.x.4:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

#节点2#

broker.id=1
listeners=PLAINTEXT://10.x.x.3:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/var/log/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.x.x.2:2181,10.x.x.3:2181,10.x.x.4:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

#节点3#

 broker.id=3
listeners=PLAINTEXT://10.x.x.4:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/var/log/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.x.x.2:2181,10.x.x.3:2181,10.x.x.4:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

4、 配置 systemd kafka服务，创建kafka服务文件(3台节点都需要执行)

cd /lib/systemd/system/

vim kafka.service

[Unit]
Description=broker(Apache Kafka server)
After=network.target zookeeper.service

[Service]
Type=simple
Environment="PATH=/usr/local/jdk1.8.0_311/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"
User=root
Group=root
ExecStart=/usr/local/kafka/kafka_2.13-3.4.0/bin/kafka-server-start.sh /usr/local/kafka/kafka_2.13-3.4.0/config/server.properties
ExecStop=/usr/local/kafka/kafka_2.13-3.4.0/bin/kafka-server-stop.sh
Restart=on-failure

[Install]
WantedBy=multi-user.target

 5、 刷新systemctl 

systemctl daemon-reload

设置kafka开机自启动

systemctl enable kafka