环境
1.软件版本
| 系统 | 版本 |
|---|---|
| centos | 7.9(内核采用4.19) |
| docker | 20.10.15 |
| kubeadm | 1.22.17 |
2.ip划分
| 主机名 | ip地址 | 系统配置 |
|---|---|---|
| kubeadm-master | 10.103.236.201 | 2core_2g |
| kubeadm-node01 | 10.103.236.202 | 1core_2g |
| kubeadm-node02 | 10.103.236.203 | 1core_2g |
| kubeadm-node03 | 10.103.236.204 | 1core_2g |
| pod网段 | 172.16.0.0/12 | |
| service网段 | 192.168.0.0/16 | |
| host网段 | 10.103.236.0/12 |
1.配置kubeadm源
1.1安装依赖
yum install -y yum-utils device-mapper-persistent-data lvm2 wget jq psmisc vim net-tools telnet
1.2配置aliyun源
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
#修改地址
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
1.3配置docker源
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
1.4配置kubernetes源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
2.基本环境配置
文档，https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
2.1 关闭selinux等
所有节点关闭防火墙、selinux、dnsmasq、swap
systemctl disable --now firewalld
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
2.2 关闭swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
2.3 时间同步
#安装服务
yum install chrony -y
#启动服务
systemctl enable chronyd --now
#同步时间
[root@kube-master yum.repos.d]# chronyc sources
210 Number of sources = 4
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
^- tock.ntp.infomaniak.ch 1 10 377 283 -10ms[ -10ms] +/- 87ms
^- ntp8.flashdance.cx 2 10 377 192 -18ms[ -18ms] +/- 154ms
^* 139.199.215.251 2 10 377 859 -902us[ -511us] +/- 59ms
^- tick.ntp.infomaniak.ch 1 10 377 27m +26ms[ +26ms] +/- 133ms
- 或者
安装ntpdate
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y
#操作如下
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
2.4 limit配置
ulimit -SHn 65535
#添加配置文件
vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
2.5 内核配置
1.升级内核
CentOS7 需要升级内核至4.18+，本地升级的版本为4.19，所有节点升级
#下载
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
#安装
yum localinstall -y kernel-ml*
#更改内核启动顺序
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
#查看内核启动版本
[root@kube-master yum.repos.d]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64
#启动所有节点
reboot
#查看版本
uname -a
2.安装ipvsadm
yum install ipvsadm ipset sysstat conntrack libseccomp -y
- 配置ipvs模块
在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack
在内核4.18以下使用nf_conntrack_ipv4
vim /etc/modules-load.d/ipvs.conf
# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
#启动
systemctl enable --now systemd-modules-load.service
- 或者其他方式
#启动时能自动加载模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#授权
chmod 755 /etc/sysconfig/modules/ipvs.modules
#执行
sh /etc/sysconfig/modules/ipvs.modules
#查看
[root@kube-master yum.repos.d]# lsmod |grep -e ip_vs -e nf_conntrack
nf_conntrack_netlink 40960 0
nfnetlink 16384 3 nf_conntrack_netlink,ip_set
ip_vs_ftp 16384 0
nf_nat 32768 2 nf_nat_ipv4,ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 151552 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 143360 6 xt_conntrack,nf_nat,ipt_MASQUERADE,nf_nat_ipv4,nf_conntrack_netlink,ip_vs
nf_defrag_ipv6 20480 1 nf_conntrack
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 4 nf_conntrack,nf_nat,xfs,ip_vs
3.配置必要参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
net.ipv4.conf.all.route_localnet = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
- 重新加载
sysctl --system
- 查看模块是否加载好, 如果不能加载 则尝试重启系统
reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack
2.6配置hosts
#新添加如下，对应关系错误，否则会出现calico启动错误
vim /etc/hosts
10.103.236.201 kube-master
10.103.236.202 kube-node01
10.103.236.203 kube-node02
10.103.236.204 kube-node03
2.7配置集群命令自动补全
https://kubernetes.io/zh/docs/tasks/tools/included/optional-kubectl-configs-bash-linux/
yum install bash-completion -y
echo 'source <(kubectl completion bash)' >>~/.bashrc
3.Runtime安装
❌ 注意 如果安装的版本低于1.24，选择Docker和Containerd均可，高于1.24选择Containerd作为Runtime :::
3.1 Containerd作为Runtime
所有节点
1.安装docker-ce-20.10
yum install docker-ce-20.10.* docker-ce-cli-20.10.* -y
#可以无需启动Docker，只需要配置和启动Containerd即可
2.配置Containerd所需模块
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
#加载模块
modprobe -- overlay
modprobe -- br_netfilter
3.配置Containerd所需的内核
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
#重新加载内核
sysctl --system
4.配置Containerd文件
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
找到containerd.runtimes.runc.options，添加SystemdCgroup = true（如果已存在直接修改，否则会报错）
- 修改镜像，否则你懂的
默认是， sandbox_image = "registry.k8s.io/pause:3.6",根据版本进行修改
vim /etc/containerd/config.toml
#修改成
sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6"
5.启动
systemctl daemon-reload
systemctl enable --now containerd
#配置crictl客户端连接的运行时位置
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
3.2 Docker作为Runtime
❌ 注意
如果选择Docker作为Runtime，安装步骤较Containerd较为简单，只需要安装并启动即可（版本小于1.24）
:::
1.安装
yum install docker-ce-20.10.* docker-ce-cli-20.10.* -y
新版官方建议，新版Kubelet建议使用systemd，所以把Docker的CgroupDriver也改成systemd
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": [
"native.cgroupdriver=systemd"
],
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 5,
"live-restore":true,
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file":"5"
},
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
],
"registry-mirrors" : [
],
"data-root": "/data/docker"
}
EOF
2.启动
systemctl daemon-reload && systemctl enable --now docker
4.安装Kubernetes组件
1.查看版本
- 在master上面查看最新版本
yum list kubeadm.x86_64 --showduplicates | sort -r
2.安装
所有节点安装，kubeadm、kubelet和kubectl
- 指定版本安装
yum install -y kubelet-1.22.17 kubeadm-1.22.* kubectl-1.22.17
- 安装最新
yum install kubeadm-1.22* kubelet-1.22* kubectl-1.22* -y
: ❌ 注意
如果选择的是Containerd作为的Runtime，需要更改Kubelet的配置使用Containerd作为Runtime：
:::
cgroup驱动
cat >/etc/sysconfig/kubelet<<EOF
KUBELET_KUBEADM_ARGS="--cgroup-driver=systemd --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
EOF