k8s集群二进制安装部署

标签：https 二进制 192.168 etc 集群 etcd -- k8s

1、前期规划

主机规划

IP地址	主机名	主机角色	软件列表
192.168.16.129	k8s-master01	master	kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet、haproxy、keepalived
192.168.16.130	k8s-master02	master	kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet、haproxy、keepalived
192.168.16.131	k8s-master03	master	kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet
192.168.16.132	k8s-node1	node	kubelet、kube-proxy

软件版本

软件名称	版本	备注
centos7	kernel:6.6
kubernetes	v1.21.10
etcd	v3.5.2
calico	v3.19.4
coredns	v1.8.4
docker	20.10.13	yum安装
haproxy	5.18	yum安装
keepalived	3.5	yum安装

网络地址规划

网络名称	网段	备注
Node网络	192.168.16.0/24
Service网络	10.96.0.0/16
Pod网络	10.244.0.0/16

2、所有主机通用配置

设置主机名和hosts文件解析

# cat /etc/hosts
192.168.150.184 k8s-master1
192.168.150.185 k8s-master2
192.168.150.186 k8s-master3
192.168.150.187 k8s-node1

关闭防火墙、Selinux、swap分区

设置时间同步

limit设置

# vim /etc/security/limits.conf
*    soft     nofile     655360
*    hard     nofile     131072
*    soft     nproc      655350
*    hard     nproc      655350
*    soft     memlock    unlimited
*    hard     memlock    unlimited

安装ipvs管理模块，并配置

# yum install ipvsadm ipset syssyay conntrack libseccomp -y
# modprobe -- ip_vs
# modprobe -- ip_vs_rr
# modprobe -- ip_vs_wrr
# modprobe -- ip_vs_sh
# modprobe -- nf_conntrack

# cat >/etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
# systemctl enable --now systemd-modules-load
# systemctl restart systemd-modules-load

内核升级

# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
# yum install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
# yum --enablerepo="elrepo-kernel" -y install kernel-ml.x86_64
# grub2-set-default 0
# grub2-mkconfig -o /boot/grub2/grub.cfg
# reboot

安装工具

# yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git lrzsz -y

3、高可用配置

在master01和master02主机安装haproxy和keepalived

haproxy配置

# yum install haproxy keepalived -y
# cat /etc/haproxy/haproxy.cfg 
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
        maxconn 2000
        ulimit-n 16384
        log 127.0.0.1 local0 err

defaults
        log global
        mode http
        option httplog
        timeout connect 5000
        timeout client  50000
        timeout server  50000
        timeout http-request 15s
        timeout http-keep-alive 15s



frontend monitor-in
        bind 0.0.0.0:33305
        mode http
        option httplog
        monitor-uri /monitor

frontend k8s-master
        bind 0.0.0.0:16443
        bind 127.0.0.1:16443
        mode tcp
        option tcplog
        tcp-request inspect-delay 5s
        default_backend k8s-master

backend k8s-master
        mode tcp
        option tcplog
        option tcp-check
        balance roundrobin
        default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
        server master01 192.168.16.129:6443 check
        server master02 192.168.16.130:6443 check
        server master03 192.168.16.131:6443 check

# systemctl enable --now haproxy

浏览器访问验证 http://192.168.16.130:33305/monitor

keepalived配置

# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   script_user root
   enable_script_security
}

vrrp_script chk_apiserver {
   script "/etc/keepalived/check_apiserver.sh"
   interval 5
   weight -5
   fall 2
   rise 1
}

vrrp_instance VI_1 {
    state MASTER                        ## 备机设置 BACKUP
    interface ens33
    mcast_src_ip 192.168.16.129         ## 备机设置自己的IP地址192.168.150.176
    virtual_router_id 51
    priority 101                        ## 备机设置优先级 99
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass abc123
    }
    virtual_ipaddress {
        192.168.16.250
    }
    track_script {
        chk_apiserver
    }
}
# cat /etc/keepalived/check_apiserver.sh 
#!/bin/bash

err=0
for k in $(seq 1 3)
do
    check_code=$(pgrep haproxy)
    if [[ $check_code == "" ]]; then
        err=$(expr $err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ $err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
# systemctl enable --now keepalived

配置主机ssh免密连接

4、使用cfssl工具创建证书

获取cfssl工具，实现正式签发的工具

# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 --no-check-certificate
# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 --no-check-certificate
# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 --no-check-certificate
# chmod +x cfssl*
# mv cfssl_linux-amd64 /usr/bin/cfssl
# mv cfssljson_linux-amd64 /usr/bin/cfssljson
# mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

创建CA证书

# cat ca-csr.json
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "kubemsb",
      "OU": "CN"
    }
  ],
  "ca": {
      "expiry": "87600h"
  }
}

# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2023/11/13 11:00:31 [INFO] generating a new CA key and certificate from CSR
2023/11/13 11:00:31 [INFO] generate received request
2023/11/13 11:00:31 [INFO] received CSR
2023/11/13 11:00:31 [INFO] generating key: rsa-2048
2023/11/13 11:00:31 [INFO] encoded CSR
2023/11/13 11:00:31 [INFO] signed certificate with serial number 574209306477940501530924598323722273337915651468

## 配置ca证书策略
# cat ca-config.json 
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}

5、etcd集群安装

生成etcd证书

# cat etcd-csr.json 
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.150.184",
    "192.168.150.185",
    "192.168.150.186"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "kubemsb",
      "OU": "CN"
    }
  ]
}

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

etcd集群部署

# wget https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz
# tar -zxvf etcd-v3.5.2-linux-amd64.tar.gz
# cp -p etcd-v3.5.2-linux-amd64/etcd* /usr/bin/
# mkdir /etc/etcd
# scp /usr/bin/etcd* k8s-master02:/usr/bin/
# scp /usr/bin/etcd* k8s-master03:/usr/bin/
# vim /etc/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.16.129:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.16.129:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.16.129:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.16.129:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.16.129:2380,etcd2=https://192.168.16.130:2380,etcd3=https://192.168.16.131:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"


# mkdir -p /etc/etcd/ssl
# mkdir  -p /var/lib/etcd/default.etcd
# cp ca*.pem /etc/etcd/ssl/
# cp etcd*.pem /etc/etcd/ssl/
# vim /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
  --cert-file=/etc/etcd/ssl/etcd.pem \
  --key-file=/etc/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/etcd/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
  --client-cert-auth \
  --peer-client-cert-auth
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

## 注意拷贝的etcd.conf文件修改IP地址和节点名
# scp /etc/etcd/etcd.conf k8s-master02:/etc/etcd/
# scp /etc/etcd/etcd.conf k8s-master03:/etc/etcd/
# scp /etc/etcd/ssl/* k8s-master02:/etc/etcd/ssl/
# scp /etc/etcd/ssl/* k8s-master03:/etc/etcd/ssl/
# scp /etc/systemd/system/etcd.service k8s-master02:/etc/systemd/system/
# scp /etc/systemd/system/etcd.service k8s-master03:/etc/systemd/system/
## 三台主机都需启动
# systemctl daemon-reload
# systemctl enable --now etcd
# systemctl status etcd

# ETCDCTL_API=3 /usr/bin/etcdctl --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.16.129:2379,https://192.168.16.130:2379,https://192.168.16.131:2379" endpoint health --write-out=table
+-----------------------------+--------+-------------+-------+
|          ENDPOINT           | HEALTH |    TOOK     | ERROR |
+-----------------------------+--------+-------------+-------+
| https://192.168.16.129:2379 |   true |  9.964038ms |       |
| https://192.168.16.131:2379 |   true | 10.207664ms |       |
| https://192.168.16.130:2379 |   true | 11.264541ms |       |
+-----------------------------+--------+-------------+-------+

标签：https,二进制,192.168,etc,集群,etcd,--,k8s
From： https://www.cnblogs.com/zbc230/p/17849562.html