现象
root@auto-server:/home/ogreks# kubectl get nodes
E0309 02:06:37.085370 134275 memcache.go:265] couldn't get current server API group list: Get "https://0.0.0.0:443/api?timeout=32s": tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
E0309 02:06:37.090467 134275 memcache.go:265] couldn't get current server API group list: Get "https://0.0.0.0:443/api?timeout=32s": tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
E0309 02:06:37.095054 134275 memcache.go:265] couldn't get current server API group list: Get "https://0.0.0.0:443/api?timeout=32s": tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
E0309 02:06:37.099202 134275 memcache.go:265] couldn't get current server API group list: Get "https://0.0.0.0:443/api?timeout=32s": tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
E0309 02:06:37.104604 134275 memcache.go:265] couldn't get current server API group list: Get "https://0.0.0.0:443/api?timeout=32s": tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
Unable to connect to the server: tls: failed to verify certificate: x509: cannot validate certificate for 0.0.0.0 because it doesn't contain any IP SANs
journalctl -u k3s 看起来没啥问题
root@auto-server:/var/log# journalctl -u k3s
Feb 24 12:42:24 auto-server systemd[1]: Starting Lightweight Kubernetes...
Feb 24 12:42:24 auto-server sh[54611]: + /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service
Feb 24 12:42:24 auto-server sh[54612]: Failed to get unit file state for nm-cloud-setup.service: No such file or directory
Feb 24 12:42:24 auto-server k3s[54616]: time="2024-02-24T12:42:24Z" level=info msg="Acquiring lock file /var/lib/rancher/k3s/data/.lock"
Feb 24 12:42:24 auto-server k3s[54616]: time="2024-02-24T12:42:24Z" level=info msg="Preparing data dir /var/lib/rancher/k3s/data/13f9723ffde84ba41d08658d407a523bcf32698f179c9ab30cc0534e1e5d2c1a"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Starting k3s v1.28.6+k3s2 (c9f49a3b)"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Configuring sqlite3 database connection pooling: maxIdleConns=2, maxOpenConns=0, connMaxLifetime=0s"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Configuring database table schema and indexes, this may take a moment..."
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Database tables and indexes are up to date"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Kine available at unix://kine.sock"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="generated self-signed CA certificate CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27.136225175 +0000 UTC notAfter=2034-02-21 12:42:27.1>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:admin,O=system:masters signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:k3s-supervisor,O=system:masters signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:kube-controller-manager signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 1>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:kube-scheduler signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:apiserver,O=system:masters signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-2>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:kube-proxy signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:k3s-controller signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=k3s-cloud-controller-manager signed by CN=k3s-client-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="generated self-signed CA certificate CN=k3s-server-ca@1708778547: notBefore=2024-02-24 12:42:27.144278863 +0000 UTC notAfter=2034-02-21 12:42:27.1>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=kube-apiserver signed by CN=k3s-server-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000 UT>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="generated self-signed CA certificate CN=k3s-request-header-ca@1708778547: notBefore=2024-02-24 12:42:27.14588523 +0000 UTC notAfter=2034-02-21 12:>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=system:auth-proxy signed by CN=k3s-request-header-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="generated self-signed CA certificate CN=etcd-server-ca@1708778547: notBefore=2024-02-24 12:42:27.147276722 +0000 UTC notAfter=2034-02-21 12:42:27.>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=etcd-client signed by CN=etcd-server-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000 UTC"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="generated self-signed CA certificate CN=etcd-peer-ca@1708778547: notBefore=2024-02-24 12:42:27.148638132 +0000 UTC notAfter=2034-02-21 12:42:27.14>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=etcd-peer signed by CN=etcd-peer-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000 UTC"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=etcd-server signed by CN=etcd-server-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000 UTC"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Saving cluster bootstrap data to datastore"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="certificate CN=k3s,O=k3s signed by CN=k3s-server-ca@1708778547: notBefore=2024-02-24 12:42:27 +0000 UTC notAfter=2025-02-23 12:42:27 +0000 UTC"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=warning msg="dynamiclistener [::]:6443: no cached certificate available for preload - deferring certificate load until storage initialization or first clien>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Active TLS secret / (ver=) (count 11): map[listener.cattle.io/cn-10.43.0.1:10.43.0.1 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/ranch>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --au>Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Waiting for API server to become available"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Running cloud-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/cloud-controller.kubeconf>Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.519858 54616 options.go:220] external host was not specified, using 192.168.50.100
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Server node token is available at /var/lib/rancher/k3s/server/token"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="To join server node to cluster: k3s server -s https://192.168.50.100:6443 -t ${SERVER_NODE_TOKEN}"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Agent node token is available at /var/lib/rancher/k3s/server/agent-token"
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.520711 54616 server.go:156] Version: v1.28.6+k3s2
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.520754 54616 server.go:158] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="To join agent node to cluster: k3s agent -s https://192.168.50.100:6443 -t ${AGENT_NODE_TOKEN}"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Wrote kubeconfig /etc/rancher/k3s/k3s.yaml"
Feb 24 12:42:27 auto-server k3s[54616]: time="2024-02-24T12:42:27Z" level=info msg="Run: k3s kubectl"
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.738491 54616 shared_informer.go:311] Waiting for caches to sync for node_authorizer
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.745716 54616 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,Ta>Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.745736 54616 plugins.go:161] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolu>Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.746396 54616 instance.go:298] Using reconciler: lease
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.754235 54616 handler.go:275] Adding GroupVersion apiextensions.k8s.io v1 to ResourceManager
Feb 24 12:42:27 auto-server k3s[54616]: W0224 12:42:27.754258 54616 genericapiserver.go:744] Skipping API apiextensions.k8s.io/v1beta1 because it has no resources.
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.828236 54616 handler.go:275] Adding GroupVersion v1 to ResourceManager
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.828416 54616 instance.go:709] API group "internal.apiserver.k8s.io" is not enabled, skipping.
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.928179 54616 instance.go:709] API group "resource.k8s.io" is not enabled, skipping.
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.935151 54616 handler.go:275] Adding GroupVersion authentication.k8s.io v1 to ResourceManager
Feb 24 12:42:27 auto-server k3s[54616]: W0224 12:42:27.935174 54616 genericapiserver.go:744] Skipping API authentication.k8s.io/v1beta1 because it has no resources.
Feb 24 12:42:27 auto-server k3s[54616]: W0224 12:42:27.935181 54616 genericapiserver.go:744] Skipping API authentication.k8s.io/v1alpha1 because it has no resources.
Feb 24 12:42:27 auto-server k3s[54616]: I0224 12:42:27.935573 54616 handler.go:275] Adding GroupVersion authorization.k8s.io v1 to ResourceManager
Feb 24 12:42:27 auto-server k3s[54616]: W0224 12:42:27.935592 54616 genericapiserver.go:744] Skipping API authorization.k8s.io/v1beta1 because it has no resources.
systemctl status k3s
可以看到"failed to get CA certs: https://0.0.0.0:443/cacerts: 404 Not Found"
root@auto-server:/var/log# sudo systemctl status k3s
● k3s.service - Lightweight Kubernetes
Loaded: loaded (/etc/systemd/system/k3s.service; enabled; vendor preset: enabled)
Active: active (running) since Sat 2024-03-09 02:10:32 UTC; 12min ago
Docs: https://k3s.io
Process: 134305 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null (code=exited, status=0/SUCCESS)
Process: 134307 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
Process: 134308 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
Main PID: 134309 (k3s-server)
Tasks: 33
Memory: 330.3M
CPU: 13min 55.097s
CGroup: /system.slice/k3s.service
└─134309 "/usr/local/bin/k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" >
Mar 09 02:23:16 auto-server k3s[134309]: time="2024-03-09T02:23:16Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:17 auto-server k3s[134309]: time="2024-03-09T02:23:17Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:17 auto-server k3s[134309]: time="2024-03-09T02:23:17Z" level=error msg="failed to get CA certs: https://0.0.0.0:443/cacerts: 404 Not Found"
Mar 09 02:23:18 auto-server k3s[134309]: time="2024-03-09T02:23:18Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:19 auto-server k3s[134309]: time="2024-03-09T02:23:19Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:19 auto-server k3s[134309]: time="2024-03-09T02:23:19Z" level=error msg="failed to get CA certs: https://0.0.0.0:443/cacerts: 404 Not Found"
Mar 09 02:23:20 auto-server k3s[134309]: time="2024-03-09T02:23:20Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:21 auto-server k3s[134309]: time="2024-03-09T02:23:21Z" level=info msg="Waiting for control-plane node agent startup"
Mar 09 02:23:21 auto-server k3s[134309]: time="2024-03-09T02:23:21Z" level=error msg="failed to get CA certs: https://0.0.0.0:443/cacerts: 404 Not Found"
Mar 09 02:23:22 auto-server k3s[134309]: time="2024-03-09T02:23:22Z" level=info msg="Waiting for control-plane node agent startup"