部署KAFKA集群
机器：
| eks-zk-kafka001 | 10.0.0.41 |
| eks-zk-kafka002 | 10.0.0.42 |
| eks-zk-kafka003 | 10.0.0.43 |
初始化单机环境
#关闭防火墙
systemctl disable --now firewalld
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager
#关闭selinux
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
#关闭swap分区，fstab注释swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
#同步机器时间
安装ntpdate
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y
所有节点同步时间。时间同步配置如下：
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
ulimit -SHn 65535
vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited 部署kafka集群
前提：已经部署完zookeeper集群
下载地址：https://kafka.apache.org/downloads
#创建目录
mkdir -p /home/work
#把tar包解压到数据目录
tar -xvf kafka_2.13-3.7.0.tgz -C /home/work
#修改配置文件
cat config/server.properties | egrep -v '^#|^$'
broker.id=0
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/home/work/kafka/logs/kafka-logs #根据自己的情况修改
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zk-kafka001:2181,zk-kafka002:2181,zk-kafka003:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0启动
./kafka-server-start.sh -daemon ../config/server.properties常用命令：
#列出所有的topic
./bin/kafka-topics.sh --list --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092
#创建一个topic
./kafka-topics.sh --create --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092 --replication-factor 1 --partitions 1 --topic testTopic
#查看指定Topic
./bin/kafka-topics.sh --describe --topic sunlixin-event-log 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092
#增加分区
bin/kafka-topics.sh 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092 --alter --topic ops_coffee --partitions 5
#删除Topic
./bin/kafka-topics.sh --delete --topic your_topic_name --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092
#启动 Kafka 控制台消费者，并从指定的主题（topic）开始消费消息
./kafka-console-consumer.sh --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092 --topic sunlixin-event-log --from-beginning
#查看Group列表
bin/kafka-consumer-groups.sh --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092 --list
#查看Group消费情况
bin/kafka-consumer-groups.sh --bootstrap-server 10.175.234.41:9092,10.176.19.12:9092,10.176.23.40:9092 --group logstash-group-sunlixin --describe标签：10.176,二进制,9092,kafka,topic,部署,--,server From： https://www.cnblogs.com/Direction-of-efforts/p/18107159