每天一点基础K8S--K8S中的ingress和ingress controller

ingress和ingress controller

1、背景

# 通过service的nodeport和clusterIP可以将后端pod服务进行代理和负载。但该代理为四层应用，即对IP和port进行代理。

# 根据之前的学习指导，K8S中每一个service资源的创建都会对应的产生一个域名，在集群内可以通过域名service-name.namespace.svc.cluster.local对后端服务进行访问。

# K8S集群中运行的业务大多数情况还是需要集群外访问的，此时有几种方案：

# 方案一：通过service的nodePort映射到node的物理网卡，此时只能通过ip+port进行访问，即四层代理；

# 方案二：在K8S集群外配置nginx服务器，将访问某一URL的流量代理到K8S集群中的service，再由service代理到后端pod。此时可以对集群外的流量实现7层代理，但是对于K8S集群中service变动频繁，需要频繁修改nginx配置文件，并手动reload，维护相对麻烦。

# 方案三： 通过K8S中的ingress和ingress controller资源，它可以根据后端ingress变化，自动配置nginx配置文件，既实现了7层代理，维护也简单。

2、ingress和ingress controller介绍

# ingress是将内部服务报错到集群外，并提供基于域名、负载均衡等服务。

# ingress-controller是负载均衡调度器，是客户端流量的入口，常用的调度器有nginx、traefik。ingress-controller就是去动态检测ingress的变化，自动修改nginx配置文件。

3、ingress-controller（nginx）高可用部署

3.1、部署nginx-keepalived实现高可用

# 按照前面的理解，ingress-controller是客户端流量的入口，那么在重要环境中，建议将ingress-controller进行高可用部署。

# 此处在only-worker-node上部署nginx + keepalive实现ingress- controller高可用。

# 给worker节点打上标签，便于区分。
[root@master-worker-node-1 ingress]# kubectl label nodes only-worker-node-3 kubernetes/ingress-controller=nginx
node/only-worker-node-3 labeled
[root@master-worker-node-1 ingress]# kubectl label nodes only-worker-node-4 kubernetes/ingress-controller=nginx
node/only-worker-node-4 labeled

# 按照nginx和keepalive实现ingress controller nginx高可用，按照nginx和keepalived
[root@only-worker-node-3 ~]# yum install -y nginx keepalived
[root@only-worker-node-4 ~]# yum install -y nginx keepalived

# 修改nginx配置文件
# 四层负载均衡，为2台nginx-ingress-controller组件提供负载均衡，两个worker的nginx配置文件相同。
stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream nginx-ingress-controller {
       server 192.168.122.132:80 weight=5 max_fails=3 fail_timeout=30s;   # nginx ingress controller 1 IP:PORT
       server 192.168.122.182:80 weight=5 max_fails=3 fail_timeout=30s;   # nginx ingress controller 2 IP:PORT
    }

    server {
       listen 1080; 
       proxy_pass nginx-ingress-controller;
    }
}
[root@only-worker-node-3 nginx]# scp nginx.conf node4:/etc/nginx/nginx.conf
nginx.conf                                                                                                          100% 1393   139.3KB/s   00:00    
[root@only-worker-node-3 nginx]# ssh node4 systemctl enable nginx --now
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.

[root@only-worker-node-3 ~]# netstat -tnlupa |  grep nginx 
tcp        0      0 0.0.0.0:80           0.0.0.0:*               LISTEN      827/nginx: master p 
tcp        0      0 0.0.0.0:1080           0.0.0.0:*               LISTEN      827/nginx: master p 

# 配置keepalived
[root@only-worker-node-3 keepalived]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     [email protected]
     [email protected]
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id nginx-ingress-slave    
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
vrrp_script check_nginx{
   script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
    state MASTER  # vrrp角色
    interface ens3   # 网卡信息
    virtual_router_id 100  # node 3 4 router id 相同
    priority 100  # 优先级，node 3 4 不相同
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.100.252  # VIP
    }
    track_script{
        check_nginx  # 监测脚本
    }
}

# 测试keepalive状态切换
[root@only-worker-node-3 keepalived]# ip add  |  grep 192.168
    inet 192.168.122.132/24 brd 192.168.122.255 scope global noprefixroute ens3
    inet 192.168.122.252/32 scope global ens3
[root@only-worker-node-3 keepalived]# systemctl stop keepalived.service 
[root@only-worker-node-3 keepalived]# ip add  |  grep 192.168
    inet 192.168.122.132/24 brd 192.168.122.255 scope global noprefixroute ens3
[root@only-worker-node-3 keepalived]# systemctl start  keepalived.service 
[root@only-worker-node-3 keepalived]# ip add  |  grep 192.168
    inet 192.168.122.132/24 brd 192.168.122.255 scope global noprefixroute ens3
    inet 192.168.122.252/32 scope global ens3
    
# 上面完成了nginx-ingress-controller入口高可用部署，下面开始部署ingress- controller

3.2、ingress-controller部署

# ingress-controller采用的控制器为deployment，两副本。配置pod反亲和性
[root@master-worker-node-1 plugin]# kubectl get all -n ingress-nginx
NAME                                            READY   STATUS      RESTARTS   AGE
pod/ingress-nginx-admission-create-fh44f        0/1     Completed   0          25h
pod/ingress-nginx-admission-patch-r2d7f         0/1     Completed   0          25h
pod/ingress-nginx-controller-64bdc78c96-9z7qk   1/1     Running     0          25h
pod/ingress-nginx-controller-64bdc78c96-wbcsw   1/1     Running     0          25h

NAME                                         TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)                      AGE
service/ingress-nginx-controller             NodePort    10.99.152.0   <none>        80:31729/TCP,443:32480/TCP   25h
service/ingress-nginx-controller-admission   ClusterIP   10.99.69.38   <none>        443/TCP                      25h

NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/ingress-nginx-controller   2/2     2            2           25h

NAME                                                  DESIRED   CURRENT   READY   AGE
replicaset.apps/ingress-nginx-controller-64bdc78c96   2         2         2       25h

NAME                                       COMPLETIONS   DURATION   AGE
job.batch/ingress-nginx-admission-create   1/1           11s        25h
job.batch/ingress-nginx-admission-patch    1/1           10s        25h

4、搭建tomcat后端服务

---
apiVersion: v1
kind: Service
metadata:
  name: tomcat
  namespace: default
spec:
  selector:
    app: tomcat
    release: canary
  ports:
  - name: http
    targetPort: 8080
    port: 8080
  - name: ajp
    targetPort: 8009
    port: 8009
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: tomcat
      release: canary
  template:
    metadata:
      labels:
        app: tomcat
        release: canary
    spec:
      containers:
      - name: tomcat
        image: tomcat:8.5.34-jre8-alpine 
        imagePullPolicy: IfNotPresent  
        ports:
        - name: http
          containerPort: 8080
          name: ajp
          containerPort: 8009

[root@master-worker-node-1 plugin]# kubectl get service -o wide 
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)             AGE     SELECTOR
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP             3d22h   <none>
tomcat       ClusterIP   10.110.26.143   <none>        8080/TCP,8009/TCP   25h     app=tomcat,release=canary
[root@master-worker-node-1 plugin]# kubectl get deployment -o wide 
NAME            READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS   IMAGES                      SELECTOR
tomcat-deploy   2/2     2            2           25h   tomcat       tomcat:8.5.34-jre8-alpine   app=tomcat,release=canary
[root@master-worker-node-1 plugin]# kubectl get pods -o wide 
NAME                             READY   STATUS    RESTARTS   AGE   IP             NODE                 NOMINATED NODE   READINESS GATES
tomcat-deploy-64d6489dd9-gn88s   1/1     Running   0          25h   10.244.54.11   only-worker-node-4   <none>           <none>
tomcat-deploy-64d6489dd9-nqf98   1/1     Running   0          25h   10.244.31.20   only-worker-node-3   <none>           <none>

# 先测试一下集群内访问
[root@master-worker-node-1 ingress]# curl -I 10.110.26.143:8080
HTTP/1.1 200 
Content-Type: text/html;charset=UTF-8
Transfer-Encoding: chunked
Date: Wed, 11 Jan 2023 15:30:34 GMT

5、使用ingress代理后端http tomcat资源

5.1、创建ingress规则

[root@master-worker-node-1 ingress]# cat ingress-tomcat.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: test-ingress-tomcat
spec:
  rules:
    - host: tomcat.test.com # 测试通过域名访问
      http:
        paths:
        - path: /   # 指定访问路径
          pathType: Prefix
          backend:  # 指定后段服务
            service:
              name: tomcat
              port:
                number: 8080
                
# 应用规则
[root@master-worker-node-1 ingress]# kubectl apply -f ingress-tomcat.yaml 
ingress.networking.k8s.io/test-ingress-tomcat created
[root@master-worker-node-1 ingress]# kubectl get ingress 
NAME            CLASS    HOSTS             ADDRESS                           PORTS   AGE
ingress-myapp   <none>   tomcat.test.com   192.168.122.132,192.168.122.182   80      25h

# 查看ingress 规则
[root@master-worker-node-1 ingress]# kubectl describe ingress ingress-myapp 
Name:             ingress-myapp
Labels:           <none>
Namespace:        default
Address:          192.168.122.132,192.168.122.182 # 表示两个ingress- controller的地址
Ingress Class:    <none>
Default backend:  <default>
Rules:
  Host             Path  Backends
  ----             ----  --------
  tomcat.test.com  
                   /   tomcat:8080 (10.244.31.20:8080,10.244.54.11:8080)
Annotations:       kubernetes.io/ingress.class: nginx  # 正常情况下还需配置defaultbackend
Events:            <none>

5.2、测试集群外访问

# 修改主机的地址
[23:44:28 remote-server root ~] # cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
122.112.148.100 huawei
120.48.27.104 bd1
192.168.122.252 tomcat.test.com

# 测试访问
[23:44:35 remote-server root ~] # curl -I tomcat.test.com:1080
HTTP/1.1 200 
Date: Wed, 11 Jan 2023 15:34:37 GMT
Content-Type: text/html;charset=UTF-8
Connection: keep-alive

5.3、业务流量说明

# 流量大致走向：
# 1、客户端根据主机将tomcat.test.com解析为192.168.122.252，访问192.168.122.252：1080。client --> 192.168.122.252:1080

# 2、根据keepalive优先级配置，192.168.122.252地址在节点only-worker-node-3上，1080端口为nginx服务，根据nginx配置文件将流量负载到某一台实服务器，此处以192.168.122.132(only-worker-node-3)为例。--> 192.168.122.132:80

# 3、由于192.168.122.132:80端口为nginx-ingress-controller deployment中pod暴露的端口。流量经某一nginx-ingress-controller pod处理，pod中的nginx.conf根据ingress 规则中的host信息，将访问tomcat.test.com的流量交给后端tomcat service的8080端口处理。此处的backend信息可以安装ingress-nginx plugin查看。--> [service]tomcat:8080

# 4、service收到流量后，根据负载规则（默认rr）负载到后端pod，--> tomcat-pod

6、总结

1、service只能用于ip+port的负载，即4层负载。K8S内资源只能通过nodeport暴露到集群外，为了实现高可用，需要通过类似nginx-keepalive实现高可用，但是nginx的配置文件的维护工作在相对频繁变化的K8S service下比较麻烦。

2、为了实现nginx配置文件自动生成，自动加载。通过nginx-ingress-controller实现。并且可以实现7层负载。