问题现象
1.当我们使用kubectl命令时发现报如下错误
[root@k8s-master ~]# kubectl get pods
The connection to the server 192.168.75.200:6443 was refused - did you specify the right host or port?
[root@k8s-master ~]#
2.查看6443端口没有被监听，重启几次kubelet服务后，还是报错如下，6443端口还是监听不起来
[root@k8s-master ~]# systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node Agent
Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Drop-In: /usr/lib/systemd/system/kubelet.service.d
└─10-kubeadm.conf
Active: active (running) since 一 2023-02-06 13:23:31 CST; 3s ago
Docs: https://kubernetes.io/docs/
Main PID: 13928 (kubelet)
Tasks: 20
Memory: 81.1M
CGroup: /system.slice/kubelet.service
├─13928 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --network-plugi...
└─14149 /opt/cni/bin/calico
2月 06 13:23:34 k8s-master kubelet[13928]: E0206 13:23:34.687235 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:34 k8s-master kubelet[13928]: E0206 13:23:34.787527 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:34 k8s-master kubelet[13928]: E0206 13:23:34.888927 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:34 k8s-master kubelet[13928]: E0206 13:23:34.989054 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.089991 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.191188 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.292653 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.399606 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.499736 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
2月 06 13:23:35 k8s-master kubelet[13928]: E0206 13:23:35.600198 13928 kubelet.go:2407] "Error getting node" err="node \"k8s-master\" not found"
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# ss -ntpl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 16384 127.0.0.1:10257 *:* users:(("kube-controller",pid=2300,fd=7))
LISTEN 0 16384 127.0.0.1:10259 *:* users:(("kube-scheduler",pid=2222,fd=7))
LISTEN 0 128 *:22 *:* users:(("sshd",pid=1079,fd=3))
LISTEN 0 16384 127.0.0.1:42683 *:* users:(("kubelet",pid=7785,fd=15))
LISTEN 0 16384 127.0.0.1:10248 *:* users:(("kubelet",pid=7785,fd=32))
LISTEN 0 16384 192.168.75.200:2379 *:* users:(("etcd",pid=1937,fd=9))
LISTEN 0 16384 127.0.0.1:2379 *:* users:(("etcd",pid=1937,fd=8))
LISTEN 0 16384 192.168.75.200:2380 *:* users:(("etcd",pid=1937,fd=7))
LISTEN 0 16384 127.0.0.1:2381 *:* users:(("etcd",pid=1937,fd=14))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1079,fd=4))
LISTEN 0 16384 [::]:10250 [::]:* users:(("kubelet",pid=7785,fd=37))
[root@k8s-master ~]#
[root@k8s-master ~]#
3.rm重启apiserver容器还是不解决问题
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# docker ps | grep api
5cba441ac2de 838d692cbe28 "kube-apiserver --ad…" 5 seconds ago Up 3 seconds k8s_kube-apiserver_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_49
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 4 minutes ago Up 4 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# docker rm 5cba441ac2de
5cba441ac2de
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]# docker rm 85e4c8b70cd7
Error response from daemon: You cannot remove a running container 85e4c8b70cd7f64e5854ab77f44499a1079a2c6e7a4f81dcad9d964819cff68a. Stop the container before attempting removal or force remove
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# docker ps | grep api
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 5 minutes ago Up 5 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]#
[root@k8s-master ~]#
[root@k8s-master ~]# docker ps | grep api
79849b038174 838d692cbe28 "kube-apiserver --ad…" 15 seconds ago Up 14 seconds k8s_kube-apiserver_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_52
85e4c8b70cd7 registry.aliyuncs.com/google_containers/pause:3.5 "/pause" 7 minutes ago Up 7 minutes k8s_POD_kube-apiserver-k8s-master_kube-system_3cec10fe0cd60fe1781e7e8c9e98bd26_10
[root@k8s-master ~]#
[root@k8s-master ~]#
更新k8s集群证书
1. 查看kubeadm的版本
[root@k8s-master pki]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.1", GitCommit:"632ed300f2c34f6d6d15ca4cef3d3c7073412212", GitTreeState:"clean", BuildDate:"2021-08-19T15:44:22Z", GoVersion:"go1.16.7", Compiler:"gc", Platform:"linux/amd64"}
2.查看证书到期
openssl x509 -in kubernetes.pem -text -noout
openssl x509 -in etcd.pem -text -noout
openssl x509 -in kubernetes.pem -text -noout
openssl x509 -in kube-proxy.pem -text -noout
kubeadm安装的用如下查询
[root@k8s-master ~]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[check-expiration] Error reading configuration from the Cluster. Falling back to default configuration
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Sep 03, 2022 08:02 UTC <invalid> no
apiserver Sep 03, 2022 08:02 UTC <invalid> ca no
apiserver-etcd-client Sep 03, 2022 08:02 UTC <invalid> etcd-ca no
apiserver-kubelet-client Sep 03, 2022 08:02 UTC <invalid> ca no
controller-manager.conf Sep 03, 2022 08:02 UTC <invalid> no
etcd-healthcheck-client Sep 03, 2022 08:02 UTC <invalid> etcd-ca no
etcd-peer Sep 03, 2022 08:02 UTC <invalid> etcd-ca no
etcd-server Sep 03, 2022 08:02 UTC <invalid> etcd-ca no
front-proxy-client Sep 03, 2022 08:02 UTC <invalid> front-proxy-ca no
scheduler.conf Sep 03, 2022 08:02 UTC <invalid> no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Sep 01, 2031 08:02 UTC 8y no
etcd-ca Sep 01, 2031 08:02 UTC 8y no
front-proxy-ca Sep 01, 2031 08:02 UTC 8y no
[root@k8s-master ~]#
3.renew证书
[root@k8s-master pki]# sudo kubeadm certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[renew] Error reading configuration from the Cluster. Falling back to default configuration
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
Done renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.
[root@k8s-master pki]#
4.查看证书有效期
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]# sudo kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[check-expiration] Error reading configuration from the Cluster. Falling back to default configuration
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Feb 06, 2024 05:54 UTC 364d no
apiserver Feb 06, 2024 05:54 UTC 364d ca no
apiserver-etcd-client Feb 06, 2024 05:54 UTC 364d etcd-ca no
apiserver-kubelet-client Feb 06, 2024 05:54 UTC 364d ca no
controller-manager.conf Feb 06, 2024 05:54 UTC 364d no
etcd-healthcheck-client Feb 06, 2024 05:54 UTC 364d etcd-ca no
etcd-peer Feb 06, 2024 05:54 UTC 364d etcd-ca no
etcd-server Feb 06, 2024 05:54 UTC 364d etcd-ca no
front-proxy-client Feb 06, 2024 05:54 UTC 364d front-proxy-ca no
scheduler.conf Feb 06, 2024 05:54 UTC 364d no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Sep 01, 2031 08:02 UTC 8y no
etcd-ca Sep 01, 2031 08:02 UTC 8y no
front-proxy-ca Sep 01, 2031 08:02 UTC 8y no
[root@k8s-master pki]#
问题得到解决
[root@k8s-master pki]#
[root@k8s-master pki]# ss -ntpl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 16384 127.0.0.1:10257 *:* users:(("kube-controller",pid=2300,fd=7))
LISTEN 0 16384 127.0.0.1:10259 *:* users:(("kube-scheduler",pid=2222,fd=7))
LISTEN 0 128 *:22 *:* users:(("sshd",pid=1079,fd=3))
LISTEN 0 16384 127.0.0.1:43103 *:* users:(("kubelet",pid=35893,fd=14))
LISTEN 0 16384 127.0.0.1:10248 *:* users:(("kubelet",pid=35893,fd=31))
LISTEN 0 16384 192.168.75.200:2379 *:* users:(("etcd",pid=1937,fd=9))
LISTEN 0 16384 127.0.0.1:2379 *:* users:(("etcd",pid=1937,fd=8))
LISTEN 0 16384 192.168.75.200:2380 *:* users:(("etcd",pid=1937,fd=7))
LISTEN 0 16384 127.0.0.1:2381 *:* users:(("etcd",pid=1937,fd=14))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1079,fd=4))
LISTEN 0 16384 [::]:10250 [::]:* users:(("kubelet",pid=35893,fd=23))
LISTEN 0 16384 [::]:6443 [::]:* users:(("kube-apiserver",pid=35677,fd=7))
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]# kubectl get pods
No resources found in default namespace.
[root@k8s-master pki]#
[root@k8s-master pki]# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
istio-system istio-egressgateway-7d6f9f54d7-mwxq6 0/1 Pending 0 172d
istio-system istio-egressgateway-7d6f9f54d7-wc7pl 0/1 Terminating 0 217d
istio-system istio-ingressgateway-5d95b48945-5bcvx 0/1 Terminating 0 217d
istio-system istio-ingressgateway-5d95b48945-hqk28 0/1 Pending 0 172d
istio-system istiod-d8576dfdf-5z9lm 0/1 Pending 0 172d
istio-system istiod-d8576dfdf-qmdn2 1/1 Terminating 0 217d
kube-system calico-kube-controllers-cf4844b67-rzg4x 1/1 Running 7 (300d ago) 517d
kube-system calico-node-7vkgm 0/1 Running 8 (300d ago) 520d
kube-system calico-node-pl5t8 1/1 Running 5 (300d ago) 520d
kube-system calico-node-r2b8v 1/1 Running 6 (300d ago) 520d
kube-system coredns-7f6cbbb7b8-hlq65 1/1 Running 7 (300d ago) 517d
kube-system coredns-7f6cbbb7b8-mrdtr 1/1 Running 7 (300d ago) 517d
kube-system etcd-k8s-master 1/1 Running 8 (300d ago) 520d
kube-system kube-apiserver-k8s-master 1/1 Running 8 (300d ago) 300d
kube-system kube-controller-manager-k8s-master 1/1 Running 8 (300d ago) 520d
kube-system kube-proxy-7xjnw 1/1 Running 6 (300d ago) 520d
kube-system kube-proxy-gpz69 1/1 Running 8 (300d ago) 520d
kube-system kube-proxy-scpz6 1/1 Running 5 (300d ago) 520d
kube-system kube-scheduler-k8s-master 1/1 Running 8 (300d ago) 520d
[root@k8s-master pki]#
[root@k8s-master pki]# ll
总用量 56
-rw-r--r-- 1 root root 1285 2月 6 13:54 apiserver.crt
-rw-r--r-- 1 root root 1155 2月 6 13:54 apiserver-etcd-client.crt
-rw------- 1 root root 1679 2月 6 13:54 apiserver-etcd-client.key
-rw------- 1 root root 1675 2月 6 13:54 apiserver.key
-rw-r--r-- 1 root root 1164 2月 6 13:54 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 2月 6 13:54 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1099 9月 3 2021 ca.crt
-rw------- 1 root root 1675 9月 3 2021 ca.key
drwxr-xr-x 2 root root 162 9月 3 2021 etcd
-rw-r--r-- 1 root root 1115 9月 3 2021 front-proxy-ca.crt
-rw------- 1 root root 1675 9月 3 2021 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 2月 6 13:54 front-proxy-client.crt
-rw------- 1 root root 1675 2月 6 13:54 front-proxy-client.key
-rw------- 1 root root 1679 9月 3 2021 sa.key
-rw------- 1 root root 451 9月 3 2021 sa.pub
[root@k8s-master pki]#
[root@k8s-master pki]#