【生产】kafka 统计脚本

kafka 根据时间统计topic 是否有数据

#!/bin/sh

for x in `cat /home/kafka/kafka_2.11-1.0.2/bin/topic.list_bak`

do


brokers="192.168.27.13:9092"

#topic="cmp_ddr_topic"

path="/home/kafka/kafka_2.11-1.0.2/bin/"


#y1=`date -d "24 hour ago" +"%Y-%m-%d %H:%M:%S"`

y1=`date --date='3 months ago' "+%Y-%m-%d %H:%M:%S"`

y2=`date "+%Y-%m-%d %H:%M:%S"`

yesterday1=`date -d "${y1}" +%s000`

yesterday2=`date -d "${y2}" +%s000`


last=0

now=0

speed=0


    echo "【-------开始统计，当前统计时间为`date`----】"

    last=$(${path}/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list ${brokers} --topic $x --time ${yesterday1} |  awk -F ":" '{sum1+=$NF} END {print sum1}')

    now=$(${path}/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list ${brokers} --topic $x --time ${yesterday2} |  awk -F ":" '{sum+=$NF} END {print sum}')


    echo "数据量：`expr $now - $last`"

    echo $last

    echo $now

    sqwe=$( expr $now - $last)

    if [ $sqwe -gt 0  ]

      then

        echo $x 有数据量    
        echo $x 有数据量 >> test3

    else

        echo $x 没数据量    
        echo $x 没数据量  >> test3  
  

    fi  
done

kafka 统计topic 占用的磁盘空间

（脚本思路 --- 统计topic 分区在磁盘的占用空间  + 多节磁盘空间总和）

各节点运行一下脚本 求出各节点占用信息

#!/bin/sh

for x in `cat /home/kafka/kafka_2.11-1.0.2/bin/topic.list_bak2`

do





   a=$(du -s /data1/kafka-logs2/$x* | awk '{print $1 }'| awk '{sum+=$1}END{print sum}')

   sum=$(($a / 1000))M


   echo  $x $sum

    

  


done



====================================

#!/bin/bash

for x in `cat /root/topiclist`


do


    a=$(grep $x  zonghe.txt  |awk '{print $2 }'|sed "s/M//g"|awk '{sum+=$1}END{print sum}')

    echo $x $a M


done

topic 生命周期统计

#!/bin/sh

for x in `cat /home/kafka/kafka_2.11-1.0.2/bin/2.txt`

do


path="/home/kafka/kafka_2.11-1.0.2/bin/"


zookeeper="192.168.27.13:2181,192.168.26.22:2181,192.168.27.205:2181"



    echo "【-------开始统计，当前统计时间为`date`----】"

    last=$(${path}/kafka-configs.sh --zookeeper $zookeeper --describe --entity-name $x --entity-type topics )


    echo $x  
    echo $last


done