Kubernetes脚本——检查K8S组件/服务/配置/POD

#!/bin/bash

#echo "运维账号是否有执行常用kubectl运维命令的权限，期望结果：输出/apps/bin/kubelet-----------------------------------------" >k8s_check_result.txt
#ansible -i ./hosts k8s -m shell -a "for i in \`sudo -l\`;do echo \$i|grep -Ei kubectl;done" >> k8s_check_result.txt
#echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s集群kubectl客户端版本检查，期望结果：版本要1.13.2以上-------------------------------------------------------------">k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl version -o yaml|grep server -A 5 |grep -i gitVersion|awk -F ':' '{print \$2}'" >> k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s集群组件状态是否正常检查，期望结果：输出数值5---------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a 'sudo /apps/bin/kubectl get cs |grep -i health |wc -l'>>k8s_check_result.txt
echo -e "-----------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "etcd的配置文件检查，期望结果：输出etcd-------------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a 'ls /apps/conf/ |grep etcd'>>k8s_check_result.txt
echo -e "-----------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "etcd的数据目录检查，期望结果：输出etcd-------------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "ls /apps/data/ |grep etcd">>k8s_check_result.txt
echo -e "-----------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "etcd集群状态检查，期望结果：集群状态返回健康-------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/etcdctl --endpoint 'https://127.0.0.1:2379' --ca-file /apps/conf/kubernetes/ssl/etcd/ca.pem --cert-file \`sudo cat /apps/conf/etcd/etcd.env|grep -i etcd_cert_file|awk -F '=' '{print \$2}'\`  --key-file \`sudo cat /apps/conf/etcd/etcd.env|grep -i etcd_key_file|awk -F '=' '{print \$2}'\`  cluster-health |grep 'cluster is healthy'">>k8s_check_result.txt
echo -e "-----------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "etcd是否配置开机自启动检查，期望结果：enabled------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "systemctl list-unit-files |grep etcd.service  |awk '{print \$2}'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s管理集群的pod是否正常检查，期望结果：返回为空，则正常-------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a 'sudo /apps/bin/kubectl get pod -n kube-system -o wide |grep -v  Running'>>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s管理集群的deployment是否正常检查，期望结果：当前运行实例数量（AVAILABLE）符合预期数量（ready）--------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a 'sudo /apps/bin/kubectl get deploy -n kube-system'>>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s管理集群的ds是否正常检查，期望结果：输出为0，当前运行实例数量（AVAILABLE）符合预期数量（DESIRED）--------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl get ds -n kube-system|awk '{print \$2,\$6}'|awk 'NR>1{a+=\$1;b+=\$2};END {print a-b}'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "kubelet状态检查，期望结果：active(running)---------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "systemctl status kubelet|grep -i active |awk  '{print \$2\$3}'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "metrics是否正常检查，期望结果：1 输出均为running;2 输出200-----------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl get pod -n kube-system|grep metrics">>k8s_check_result.txt
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "curl -i -k --tlsv1 https://localhost:6443/apis/autoscaling/ -o /dev/null -s -w '%{http_code}\n'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s node节点检查，期望结果：redady状态是正常状态，notready表示状态异常-----------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl get nodes">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "k8s配置文件检查，期望结果：检查采用的配置文件以及启动服务所需要的YAML配置文件统一存在此目录下------------------------">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "sudo ls -trl /apps/conf/kubernetes">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "节点coredns组件运行状态检查，期望结果：状态正常----------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl get pod -n kube-system |grep coredns">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "节点calico组件运行状态及其控制器运行状态检查，期望结果：输出均为running----------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8smaster -m shell -a "sudo /apps/bin/kubectl get po -n kube-system |grep calico">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "隧道接口与cni配置的mtu值检查，期望结果：mtu值为1420------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "/usr/sbin/ip a |grep tunl0@|awk '{print \$4,\$5}'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

#echo "kube-system域下pod的健康检查，期望结果：所有pod都添加基础探针：livenessProbe、readnessProbe--------------------------">>k8s_check_result.txt
#for i in `sudo /apps/bin/kubectl get po -n kube-system |awk '{print $1}'|sed -n '2,50p'`; do echo $i ;  sudo /apps/bin/kubectl get po $i -n kube-system -o yaml|grep -Ei 'readnessprobe|livenessprobe';done >> k8s_check_result.txt
#echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "联通性检查，期望结果：200--------------------------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "curl -i -k --tlsv1 https://localhost:6443/apis/autoscaling/ -o /dev/null -s -w '%{http_code}\n'">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt

echo "kubelet开机自启动检查,期望结果：enabled------------------------------------------------------------------------------">>k8s_check_result.txt
ansible -i ./hosts k8s -m shell -a "systemctl list-unit-files |grep kubelet">>k8s_check_result.txt
echo -e "------------------------------------------------------------------------------------------------------------------\n\n">>k8s_check_result.txt