1.安装部署ingress-nginx
本次部署使用了高可用的形式,会在每个node节点做亲和性(master不部署),让每一个pod都部署上去,然后加入NGINX去过负载,这样我们之后用NGINX的80端口访问域名就可以了。
主机 | 地址 | 端口 |
---|---|---|
k8s-node01 | 192.168.80.48 | nginx启动端口:3080,负载均衡端口:根据ingress svc自己生成的NodePort的端口 |
k8s-node02 | 192.168.80.49 | nginx启动端口:3080,负载均衡端口:根据ingress svc自己生成的NodePort的端口 |
vip | 192.168.80.66 | 访问端口:80 |
通过 keepalived
+nginx
实现 nginx-ingress-controller
高可用。
1.1.替换镜像
查看当前版api版本
kubectl explain Ingress
KIND: Ingress
VERSION: networking.k8s.io/v1
....
注:查看ingress和自己本地的k8s版本是否对应上,在GitHub上有表格参考。
mkdir -p /root/ingress && cd /root/ingress
curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.6.4/deploy/static/provider/baremetal/deploy.yaml -O deploy.yaml
cat deploy.yaml | grep image:
image: registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f
# 替换镜像
sed -i 's#registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f#registry.cn-hangzhou.aliyuncs.com/image-storage/controller:v1.6.4#' deploy.yaml
sed -i 's#registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f#registry.cn-hangzhou.aliyuncs.com/image-storage/kube-webhook-certgen:v20220916-gd32f8c343#' deploy.yaml
注:从外网下载的镜像放在了自己的阿里镜像服务内。
1.2.ingress高可用配置
1.2.1修改文件和主机打标签
vim deploy.yaml
kind: Deployment #改为DaemonSet控制器
spec:
template:
spec:
nodeSelector: #修改节点选择,亲和度
custom/ingress-controller-ready: true
apiVersion: v1
kind: Service
metadata:
name: ingress-nginx-controller
namespace: ingress-nginx
....
spec:
....
type: NodePort #后端svc访问改成NodePort
# node主机打标签
kubectl label nodes k8s-node01 custom/ingress-controller-ready=true
kubectl label nodes k8s-node02 custom/ingress-controller-ready=true
kubectl taint nodes k8s-master01 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master02 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master03 node-role.kubernetes.io/master=true:NoSchedule
1.2.2部署ingress
# 部署ingress
kubectl apply -f deploy.yaml
# 查看ingress pod
kubectl get pod -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-admission-create-849p4 0/1 Completed 0 61s
ingress-nginx-admission-patch-7qk5c 0/1 Completed 0 61s
ingress-nginx-controller-vblkz 1/1 Running 0 61s
ingress-nginx-controller-zmn49 1/1 Running 0 61s
2.部署NGINX和keepalived
-
node01、node02操作
apt install nginx keepalived -y
sudo useradd nginx -G www-data
2.1.修改配置
# 修改默认端口为3080
cd /etc/nginx/sites-enabled
cat default
listen 3080 default_server;
listen [::]:3080 default_server;
# 重启nginx
systemctl restart nginx.service
netstat -lntup | grep 3080
tcp 0 0 0.0.0.0:3080 0.0.0.0:* LISTEN 263469/nginx: maste
tcp6 0 0 :::3080 :::* LISTEN 263469/nginx: maste
# 查看ingress本地端口
kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller NodePort 10.101.53.92 <none> 80:30371/TCP,443:32335/TCP 11m
ingress-nginx-controller-admission ClusterIP 10.101.103.120 <none> 443/TCP 11m
2.2.添加负载
cd /etc/nginx ; cp nginx.conf nginx.conf_bak
cat > nginx.conf <<"EOF"
load_module /usr/lib/nginx/modules/ngx_stream_module.so;
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main;
upstream ingress {
server 192.168.80.48:30371; # #这里配置成要访问的地址
server 192.168.80.49:30371;
}
server {
listen 80; #需要监听的端口
proxy_pass ingress;
}
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
EOF
# 检查格式
nginx -t
2.3.keepalived配置
-
k8s-node01
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
# 检查脚本
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_NGINX {
state MASTER
interface ens33 # 修改为实际网卡名
virtual_router_id 51 # VRRP 路由 ID实例,每个实例是唯一的
priority 100 # 优先级,备服务器设置 90
advert_int 1 # 指定VRRP 心跳包通告间隔时间,默认1秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟IP
virtual_ipaddress {
192.168.80.66/24
}
track_script {
check_nginx
}
}
EOF
-
k8s-node02
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_BACKUP
}
# 检查脚本
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_NGINX {
state BACKUP
interface ens33 # 修改为实际网卡名
virtual_router_id 51 # VRRP 路由 ID实例,每个实例是唯一的
priority 90 # 优先级,备服务器设置 90
advert_int 1 # 指定VRRP 心跳包通告间隔时间,默认1秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟IP
virtual_ipaddress {
192.168.80.66/24
}
track_script {
check_nginx
}
}
EOF
keepalived 检查脚本(注意脚本内的端口是需要监听的端口):
cat > /etc/keepalived/check_nginx.sh <<"EOF"
#!/bin/bash
count=$(ss -antp |grep 80 |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
exit 1
else
exit 0
fi
EOF
useradd keepalived_script
passwd keepalived_script
chown -R keepalived_script:keepalived_script /etc/keepalived/check_nginx.sh
chmod +x /etc/keepalived/check_nginx.sh
重启服务:
systemctl daemon-reload
systemctl start nginx keepalived
systemctl enable nginx keepalived
systemctl restart keepalived.service nginx.service
3.测试ingress
3.1.pod和svc创建
mkdir -p /root/ingress ; cd /root/ingress
cat > /root/ingress/deploy-demo.yaml <<EOF
#创建service为myapp
apiVersion: v1
kind: Service
metadata:
name: myapp
namespace: default
spec:
selector:
app: myapp
release: canary
ports:
- name: http
targetPort: 80
port: 80
---
#创建后端服务的pod
apiVersion: apps/v1
kind: Deployment
metadata:
name: myapp-backend-pod
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: myapp
release: canary
template:
metadata:
labels:
app: myapp
release: canary
spec:
containers:
- name: myapp
image: ikubernetes/myapp:v2
ports:
- name: http
containerPort: 80
EOF
启动容器:
kubectl apply -f deploy-demo.yaml
kubectl get pod -l app=myapp
NAME READY STATUS RESTARTS AGE
myapp-backend-pod-9f9b5bd95-5d487 1/1 Running 0 23m
myapp-backend-pod-9f9b5bd95-k87tc 1/1 Running 0 23m
myapp-backend-pod-9f9b5bd95-vssh7 1/1 Running 0 23m
3.2.ingress创建
cat > /root/ingress/ingress-myapp.yaml <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-myapp
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: "myapp.magedu.com"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: myapp
port:
number: 80
EOF
创建ingress:
kubectl apply -f ingress-myapp.yaml
kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress-myapp <none> myapp.magedu.com 192.168.80.48,192.168.80.49 80 5m15s
注:Windows本地hosts去绑定下vip地址和ingress定义的域名。
效果: