ansible部署kubernetes(1.30)

ansible部署kubernetes(1.30)

• 操作系统使用的是ubuntu 24.04，ansible使用rocky9.2

1.规划

2.ssh免密配置

$ ssh-keygen -t ed25519
$ ssh-copy-id [email protected]
$ ssh-copy-id [email protected]
$ ssh-copy-id [email protected]
$ ssh-copy-id [email protected]

$ sudo sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config \
&& sudo sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \
&& sudo sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config \
&& rm -rf /etc/ssh/sshd_config.d/50-cloud-init.conf \
&& sudo systemctl restart sshd

3.ansible部署

#下面环境使用的是rocky9.2的系统
# step1：安装
$ yum -y install ansible-core.x86_64

# step2：配置ansible配置文件
$ vim /etc/ansible/ansible.cfg 
[defaults]
inventory=/etc/ansible/hosts
roles_path=/etc/ansible/roles
host_key_checking=False

[privilege_escalation]
become=True
become_method=sudo
become_user=root
become_ask_pass=False

# step3:使用key认证,最好使用fqdn，因为这样后续部署别的集群只需要修改hosts解析记录即可
$ vim /etc/ansible/hosts
[master]
k8s-master01

[node]
k8s-node01
k8s-node02
k8s-node03

# step4：测试
$ ansible all -m ping					

4.系统配置

# step2:主机名配置
ansible master-01 -m hostname -a "name=k8s-master01.leepongmin.com"
ansible node-01 -m hostname -a "name=k8s-node01.leepongmin.com"
ansible node-02 -m hostname -a "name=k8s-node02.leepongmin.com"
ansible node-03 -m hostname -a "name=k8s-node03.leepongmin.com"

# step3:主机名解析配置
$ ansible all -m shell -a "cat <<eof>> /etc/hosts
10.10.50.11 k8s-master01.example.com master-01
10.10.50.14 k8s-node01.example.com node-01
10.10.50.15 k8s-node02.example.com node-02
10.10.50.16 k8s-node03.example.com node-03
eof"

# 4.系统仓库配置
$ ansible all -m shell -a 'cat <<eof> /etc/apt/sources.list
deb https://mirrors.aliyun.com/ubuntu/ noble main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ noble-security main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-security main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ noble-updates main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-updates main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ noble-backports main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ noble-backports main restricted universe multiverse
eof'
$ ansible all -m shell -a 'apt clean all && apt update'

5.ntp配置

$ vim chrony-install.yaml
---
- name: Install and configure Chrony
  hosts: all
  tasks:
    - name: Install Chrony
      apt:
        name: chrony
        state: present
      when: ansible_os_family == "Debian"
    - name: Install Chrony
      yum:
        name: chrony
        state: present
      when: ansible_os_family == "RedHat"
    - name: Configure Chrony NTP servers
      copy:
        dest: /etc/chrony/chrony.conf
        content: |
          server ntp.aliyun.com iburst
    - name: Restart Chrony service
      systemd:
        name: chronyd
        state: restarted
        enabled: yes
    - name: Set the timezone using timedatectl
      command: timedatectl set-timezone Asia/Shanghai

$ ansible-ploybook chrony-install.yaml

6.禁用swap

$ ansible all -m shell -a "sed -ri 's/.*swap.*/#&/' /etc/fstab"
$ ansible all -m shell -a "swapoff -a"

7.配置内核信息

# 1.加载模块
$ ansible all -m shell -a "cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF"
		
# 2.设置内核参数
$ ansible all -m shell -a "cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF"
$ ansible all -m shell -a "sysctl --system"
		
# 3.资源优化
$ ansible all -m shell -a "cat >>/etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF"

8.ipvs配置

$ ansible all -m shell -a "sudo apt install -y ipset ipvsadm"
		
# 1.加载模块
$ ansible all -m shell -a "sudo tee /etc/modules-load.d/ipvs.conf << EOF
overlay
br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF"
$ ansible all -m shell -a "systemctl restart systemd-modules-load.service"

9.containerd

# step1:安装containerd
$ wget -P /opt https://github.com/containerd/containerd/releases/download/v1.7.17/containerd-1.7.17-linux-amd64.tar.gz
#这个在github下载很慢，因此将文件下载到ansible主机上之后拷贝到所有节点
$ vim copy-containerd.yaml
- hosts: all
  gather_facts: no
  tasks:
  - name: Synchronize files to remote host
    copy:
      src: /opt/containerd-1.7.17-linux-amd64.tar.gz
      dest: /opt/containerd-1.7.17-linux-amd64.tar.gz
$ ansible-playbook copy-containerd.yaml 

$ ansible all -m shell -a "tar xzf /opt/containerd-1.7.17-linux-amd64.tar.gz -C /usr/local/"
#$ ansible all -m shell -a "cp -r /opt/bin/* /usr/local/bin/
		
$ ansible all -m shell -a "cat > /lib/systemd/system/containerd.service << EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF"

10.runc

$ wget -P /opt https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64
$ vim copy-runc.yaml
- hosts: all
  gather_facts: no
  tasks:
  - name: Synchronize files to remote host
    copy:
      src: /opt/runc.amd64
      dest: /usr/local/bin/runc
      mode: "0755"

# step3:创建配置文件
$ ansible all -m shell -a "mkdir -p /etc/containerd"
$ ansible all -m shell -a "containerd config default | tee /etc/containerd/config.toml"
		
# step4:修改底层容器地址，使用代理就不用修改
#$ vim modify-container.yaml
- hosts: all
  gather_facts: no
  tasks:
  - name: Execute bootstrapped shell command on remote host
    become: true
    shell: |
      sed -i 's|SystemdCgroup = false|SystemdCgroup = true|g' /etc/containerd/config.toml
      sed -ri -e 's@(.*sandbox_image = ).*@\1\"registry.aliyuncs.com/google_containers/pause:3.9\"@' /etc/containerd/config.toml
  - name: Reload the systemd daemon
    become: true
    command: systemctl daemon-reload
  - name: Restart the containerd service
    become: true
    systemd:
      name: containerd
      state: restarted

11.containerd proxy

$ vim containerd-proxy.yaml
- hosts: all
  gather_facts: no
  tasks:
  - name: http-proxy
    become: true
    shell: |
      sed -i '5a Environment=HTTP_PROXY="http://10.10.50.2:7890"' /lib/systemd/system/containerd.service
      sed -i '5a Environment=HTTPS_PROXY="http://10.10.50.2:7890"' /lib/systemd/system/containerd.service
      sed -i '5a Environment="NO_PROXY=localhost,127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local"' /lib/systemd/system/containerd.service
      
  - name: Reload the systemd daemon
    become: true
    command: systemctl daemon-reload
  - name: Restart the containerd service
    become: true
    systemd:
      name: containerd
      state: restarted
  - name: Reboot
    reboot:

12.kubeadm

$ ansible all -m shell -a "apt-get update && apt-get install -y apt-transport-https"
$ ansible all -m shell -a 'curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/Release.key |
    gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/ /" |
    tee /etc/apt/sources.list.d/kubernetes.list'
    
$ ansible all -m shell -a "apt-get update"
$ ansible all -m shell -a "apt-get install -y kubelet kubeadm kubectl"

13.init

$ kubeadm init --apiserver-advertise-address=10.10.50.11 --pod-network-cidr=10.100.0.0/16 --service-cidr=10.200.0.0/16 --token-ttl=0 --image-repository registry.aliyuncs.com/google_containers --upload-certs

14.calico deployment

$ wget https://raw.githubusercontent.com/projectcalico/calico/master/manifests/calico.yaml --no-check-certificate
$ vim calico.yaml
- name: CALICO_IPV4POOL_CIDR
  value: "10.100.0.0/16"
- name: CALICO_IPV4BLOCK_SIZE
  value: "24"
- name: CALICO_IPV4POOL_IPIP
  value: "Never"

15.test pod

apiVersion: v1
kind: Pod
metadata:
  name: mypod1
spec:
  nodeName: k8s-node01.example.com
  containers:
  - name: daemonapp1
    image: busybox
    command: ["sh","-c","sleep 3600"]
---
apiVersion: v1
kind: Pod
metadata:
  name: mypod2
spec:
  nodeName: k8s-node02.example.com
  containers:
  - name: daemonapp2
    image: busybox
    command: ["sh","-c","sleep 3600"]
---
apiVersion: v1
kind: Pod
metadata:
  name: mypod3
spec:
  nodeName: k8s-node03.example.com
  containers:
  - name: daemonapp3
    image: busybox
    command: ["sh","-c","sleep 3600"]