kube-vip搭建k8s1.30.5高可用集群

实验环境

机器五台
系统：ubuntu24.04
cat /etc/hosts
192.168.0.11   jichao11  k8s-master01
192.168.0.12   jichao12  k8s-master02
192.168.0.13   jichao13  k8s-master03
192.168.0.14   jichao14  k8s-worker01
192.168.0.15   jichao15  k8s-worker02
192.168.0.200  lb.kubex.com

ubuntu系统初始化

开启root 登录权限 （ubuntu 要专门打开）

sudo vim /etc/ssh/sshd_config
PermitRootLogin yes

sudo systemctl restart sshd.service

换源

cd /etc/apt
mv sources.list sources.list.bak
mv sources.list.d/ sources.list.d.bak/
vim sources.list

https://developer.aliyun.com/mirror/ubuntu?spm=a2c6h.13651102.0.0.1ecc1b11zdVBI5


deb https://mirrors.aliyun.com/ubuntu/ noble main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ noble-security main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-security main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ noble-updates main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-updates main restricted universe multiverse

# deb https://mirrors.aliyun.com/ubuntu/ noble-proposed main restricted universe multiverse
# deb-src https://mirrors.aliyun.com/ubuntu/ noble-proposed main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ noble-backports main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-backports main restricted universe multiverse


apt-get update

关闭防火墙

#关闭防火墙，
service ufw stop
update-rc.d ufw defaults-disabled
systemctl disable ufw.service

hosts文件

vim /etc/hosts
192.168.0.11   jichao11  k8s-master01
192.168.0.12   jichao12  k8s-master02
192.168.0.13   jichao13  k8s-master03
192.168.0.14   jichao14  k8s-worker01
192.168.0.15   jichao15  k8s-worker02
192.168.0.200  lb.kubex.com

系统句柄数

ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF
ulimit -a

实际dns文件--k8s指定文件

# k8s 部署的时候是看的这个文件 

cat /run/systemd/resolve/resolv.conf

系统初始化

apt-get install lrzsz git subversion gpm unzip zip wget curl htop vim
#修改时区，同步时间
apt-get install chrony -y
mv chrony.conf chrony.conf.bak
vim /etc/chrony/chrony.conf
-----
server ntp.aliyun.com iburst
stratumweight 0
driftfile /var/lib/chrony/drift
rtcsync
makestep 10 3
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
keyfile /etc/chrony.keys
commandkey 1
generatecommandkey
logchange 0.5
logdir /var/log/chrony

-----
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' > /etc/timezone
#关闭防火墙，
service ufw stop
update-rc.d ufw defaults-disabled
## 关闭swap
swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab

克隆机器

vim /etc/hostname
vim /etc/netplan/50-cloud-init.yaml

免密登录

./fgssh -user root -hosts "jichao11 jichao12 jichao13 jichao14 jichao15" -advanced -exverify -confirm

chmod 600 /root/.ssh/config

安装ipvs转发

#系统优化
cat > /etc/sysctl.d/k8s_better.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
modprobe br_netfilter
lsmod |grep conntrack
modprobe ip_conntrack
sysctl -p /etc/sysctl.d/k8s_better.conf

##系统依赖包：
apt-get install -y conntrack ipvsadm ipset jq iptables curl sysstat wget vim net-tools git
### 开启ipvs 转发
modprobe br_netfilter
mkdir -p /etc/sysconfig/modules/
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack

containerd部署文件

下载指定版本containerd
# wget https://github.com/containerd/containerd/releases/download/v1.7.21/cri-containerd-cni-1.7.21-linux-amd.tar.gz

scp cri-containerd-cni-1.7.21-linux-amd64.tar.gz k8s-master02:/root
scp cri-containerd-cni-1.7.21-linux-amd64.tar.gz k8s-master03:/root
scp cri-containerd-cni-1.7.21-linux-amd64.tar.gz k8s-worker01:/root
scp cri-containerd-cni-1.7.21-linux-amd64.tar.gz k8s-worker02:/root



# 解压安装
 tar xvf cri-containerd-cni-1.7.21-linux-amd64.tar.gz -C /

修改containerd配置文件

 mkdir /etc/containerd
 containerd config default > /etc/containerd/config.toml
 --- 
 修改
 sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
 SystemdCgroup = true 由false修改为true

开机自启动

systemctl enable --now containerd
containerd --version

安装containerd客户端nerdctl

wget https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-1.7.6-linux-amd64.tar.gz


tar xf nerdctl-1.7.6-linux-amd64.tar.gz -C /usr/local/bin/


 which nerdctl
/usr/local/bin/nerdctl

k8s

源

# 别忘记了 把bak 改为原来的
mv /etc/apt/sources.list.d.bak/ /etc/apt/sources.list.d/


apt-get update && apt-get install -y apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/Release.key |
 gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/ /" |
 tee /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -y kubelet kubeadm kubectl


apt-cache policy kubeadm ###查看版本
# 这块我们的版本是1.30.5了，和视频的4是不一样的

apt-cache showpkg kubeadm

apt-cache madison kubeadm

apt-mark hold kubelet kubeadm kubectl ## 关闭自动更新

保证一致性

vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
systemctl enable kubelet

kube-vip准备

# 虚拟ip  注意网口
export VIP=192.168.0.200
export INTERFACE=ens32  
export KVVERSION=v0.8.0



nerdctl run -it --rm --net=host ghcr.io/kube-vip/kube-vip:$KVVERSION manifest pod \
--interface $INTERFACE \
--address $VIP \
--controlplane \
--services \
--arp \
--enableLoadBalancer \
--leaderElection | tee /etc/kubernetes/manifests/kube-vip.yaml

cat /etc/kubernetes/manifests/kube-vip.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: kube-vip
  namespace: kube-system
spec:
  containers:
  - args:
    - manager
    env:
    - name: vip_arp
      value: "true"
 - name: port
   value: "6443"
 - name: vip_nodename
   valueFrom:
     fieldRef:
       fieldPath: spec.nodeName
 - name: vip_interface
   value: ens32 ###
 - name: vip_cidr
   value: "32"
 - name: dns_mode
   value: first
 - name: cp_enable
   value: "true"
 - name: cp_namespace
   value: kube-system
 - name: svc_enable
   value: "true"
 - name: svc_leasename
   value: plndr-svcs-lock
 - name: vip_leaderelection
   value: "true"
 - name: vip_leasename
   value: plndr-cp-lock
 - name: vip_leaseduration
   value: "5"
 - name: vip_renewdeadline
   value: "3"
 - name: vip_retryperiod
   value: "1"
 - name: lb_enable
 value: "true"
 - name: lb_port
   value: "6443"
 - name: lb_fwdmethod
   value: local
 - name: address
   value: 192.168.0.200
 - name: prometheus_server
   value: :2112
 image: ghcr.io/kube-vip/kube-vip:v0.8.0
 imagePullPolicy: IfNotPresent
 name: kube-vip
 resources: {}
 securityContext:
   capabilities:
     add:
     - NET_ADMIN
     - NET_RAW
 volumeMounts:
 - mountPath: /etc/kubernetes/admin.conf
   name: kubeconfig
 hostAliases:
 - hostnames:
   - kubernetes
   ip: 127.0.0.1
 hostNetwork: true
 volumes:
 - hostPath:
     path: /etc/kubernetes/admin.conf
   name: kubeconfig
status: {}




scp /etc/kubernetes/manifests/kube-vip.yaml k8s-master02:/etc/kubernetes/manifests/
scp /etc/kubernetes/manifests/kube-vip.yaml k8s-master03:/etc/kubernetes/manifests/

k8s集群初始化文件准备

kubeadm version

# 生成配置文件 #把里面内容删除即可 
kubeadm config print init-defaults > kubeadm-config.yaml

cat kubeadm-config.yaml
## 注意yaml的格式 少一个空格都跑不起来
cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
cgroupDriver: systemd
localAPIEndpoint:
  advertiseAddress: 192.168.0.11    ##
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: jichao11  ##  
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
  certSANs:
  - lb.kubex.com
  - jichao11
  - jichao12
  - jichao13
  - jichao14
  - jichao15
  - 192.168.0.11
  - 192.168.0.12
  - 192.168.0.13
  - 192.168.0.14
  - 192.168.0.15
controlPlaneEndpoint: lb.kubex.com:6443
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
  dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.30.5  ##
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
  maxPerCore: null
  min: null
  tcpCloseWaitTimeout: null
  tcpEstablishedTimeout: null
detectLocal:
  bridgeInterface: ""
  interfaceNamePrefix: ""
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
  localhostNodePorts: null
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: true
  syncPeriod: 0s
  tcpFinTimeout: 0s
  tcpTimeout: 0s
  udpTimeout: 0s
kind: KubeProxyConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
metricsBindAddress: ""
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
winkernel:
  enableDSR: false
  forwardHealthCheckVip: false
  networkName: ""
  rootHnsEndpointName: ""
  sourceVip: ""

查看下载镜像

kubeadm config images list --image-repository registry.aliyuncs.com/google_containers




# 下载
 kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers

修改kube-vip文件

sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
 /etc/kubernetes/manifests/kube-vip.yaml

初始化集群

# 在有kubeadm-config.yaml 的目录下执行

kubeadm init --config kubeadm-config.yaml --upload-certs --v=9







 # 主节点 执行
 mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf

主节点加入

kubeadm join lb.kubex.com:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:284a4a9eaefec7e5e348405d9995e5f1c257785dc66544e6f2dc3868295edc0e \
        --control-plane --certificate-key 6872a68ef12f080d0161d452458fc4d43902ae532d2244a33132acc6096c6c7a


 mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

工作节点加入

kubeadm join lb.kubex.com:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:284a4a9eaefec7e5e348405d9995e5f1c257785dc66544e6f2dc3868295edc0e

安装网络插件calico

tar xf yaml.tar.gz

vim calico.yaml


## 注意网卡

kubectl apply -f calico.yaml

kubectl get pods -n kube-system

dashboard

kubectl apply -f recommended.yaml

kubectl apply -f dashboard-user.yaml

kubectl create token admin-user -n kubernetes-dashboard



eyJhbGciOiJSUzI1NiIsImtpZCI6ImdybjdHZW5WMlNJaUQ5VGJoUzdtNmJNUnRtYWtJRElTYTFYX2p2Q09nTzgifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzI2NjQzMDg5LCJpYXQiOjE3MjY2Mzk0ODksImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiMDM3ZjI0OTQtNzE3NC00YjJkLTkwNGMtYzBjYzA3MmJhMWE5Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiMTkwZmY1MzAtZDg0YS00NGZiLWJmMzQtYTRjMjNlYzY1Zjk0In19LCJuYmYiOjE3MjY2Mzk0ODksInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDphZG1pbi11c2VyIn0.c467Y5Kb62WVQXiUEef2iOfWx3fWwmVHA1gbBtSEGBJWZFa6c6SfhTA5T99OPfCs7ypQbTM6J3TViUdB_ax9JvjjTztOG4BR5qDiyraR2TUkGo84ysK6hx7_pL2W-vpvuAA5_BAutBrBWW-LXrgzTJF69gn-I7hS5woh7OgvIX7pcmVfqmTIdgYEp8dNqvdeebsUZj0OF4FbnvMa3_srCSPtEaba7t0cQMrk_8iCOrNQ38_5mLKgF_Jjrlrg_-YExCst6xhbYig2AIt7u85M3FJLWLxa_LkQ62Klo0hufUqVhHRExMmfZfCbeGaD2egZkXSGVPIhZaWH7b7fxPrSFw

查看端口
kubectl get svc -n kubernetes-dashboard

192.168.0.11:30001

部署Nginx

kubectl apply -f ngingx-web.yaml
kubectl get pods
kubectl get svc
192.168.0.11:31180

集群状态和调用组件