一、Kubernetes service 类型详解及案例
Kubernetes service 类型：
- ExternalName
- NodePort
- ClusterIP
- loadBalancer
1.1、ClusterIP
# kubectl explain service.spec.type
ClusterIP： 默认的类型， 用于 k8s 内部之间的服务访问， 即通过内部的 service ip 实现服务间的访问， service IP 仅可以在内部访问， 不能从外部访问。
1.2、NodePort
NodePort： 在 cluster IP 的基础之上， 通过在每个 node 节点监听一个可以指定宿主机端口(nodePort)来暴露服务， 从而允许外部 client 访问 k8s 集群中的服务， nodePort 把外部 client的请求转发至 service 进行处理，
1.3、LoadBalancer
LoadBalancer： 主要在公有云如阿里云、 AWS 上使用， LoadBalancer 构建在 nodePort 基础之上， 通过公有云服务商提供的负载均衡器将 k8s 集群中的服务暴露给集群外部的 client访问
[root@easzlab-deploy 4.Ingress-cases]# cat 0.1.loadbalancer-dashboard.yaml
kind: Service
apiVersion: v1
metadata:
namespace: kubernetes-dashboard
name: dashboard-lb
labels:
k8s-app: kubernetes-dashboard
spec:
ports:
- protocol: TCP
port: 8443
targetPort: 8443
nodePort: 30063
type: LoadBalancer
selector:
k8s-app: kubernetes-dashboard
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 0.1.loadbalancer-dashboard.yaml
service/dashboard-lb created
[root@easzlab-deploy 4.Ingress-cases]#
[root@easzlab-deploy 4.Ingress-cases]# kubectl get svc -n kubernetes-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-lb LoadBalancer 10.100.32.54 <pending> 8443:30063/TCP 114s #此时这里一直pending状态，因为lb需结合公网环境配置，私有环境无法获取外部IP
dashboard-metrics-scraper ClusterIP 10.100.217.114 <none> 8000/TCP 14d
kubernetes-dashboard NodePort 10.100.153.147 <none> 443:30000/TCP 14d
[root@easzlab-deploy 4.Ingress-cases]#
1.4、ExternalName
ExternalName： 用于将 k8s 集群外部的服务映射至 k8s 集群内部访问， 从而让集群内部的pod 能够通过固定的 service name 访问集群外部的服务， 有时候也用于将不同 namespace之间的 pod 通过 ExternalName 进行访问。
[root@easzlab-deploy 4.Ingress-cases]# cat 0.2.externalName.yaml apiVersion: v1 kind: Service metadata: name: my-external-test-name namespace: default spec: type: ExternalName #service类型 externalName: www.cyhsre.com #外部域名 [root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 0.2.externalName.yaml service/my-external-test-name created [root@easzlab-deploy 4.Ingress-cases]# [root@easzlab-deploy 4.Ingress-cases]# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.100.0.1 <none> 443/TCP 14d my-external-test-name ExternalName <none> www.cyhsre.com <none> 80s [root@easzlab-deploy 4.Ingress-cases]#
验证测试：
当在pod ping my-external-test-name时，自动跳转到外部域名
Endpoints
[root@easzlab-deploy 4.Ingress-cases]# cat 0.3.Endpoints.yaml
apiVersion: v1
kind: Service
metadata:
name: mysql-production-server-name
namespace: default
spec:
ports:
- port: 6379
---
kind: Endpoints
apiVersion: v1
metadata:
name: mysql-production-server-name
namespace: default
subsets:
- addresses:
- ip: 172.16.88.169 #这里填写redis服务所在节点
ports:
- port: 6379
[root@easzlab-deploy 4.Ingress-cases]#
测试验证
在172.16.88.169节点准备redis服务
#在172.16.88.169节点安装redis服务
[root@easzlab-k8s-nfs-01 ~]# ip r default via 172.16.88.254 dev enp1s0 proto static metric 100 172.16.88.0/24 dev enp1s0 proto kernel scope link src 172.16.88.169 metric 100 [root@easzlab-k8s-nfs-01 ~]# [root@easzlab-k8s-nfs-01 ~]# apt-get install redis [root@easzlab-k8s-nfs-01 ~]# vi /etc/redis/redis.conf [root@easzlab-k8s-nfs-01 ~]# grep ^bind /etc/redis/redis.conf bind 0.0.0.0 ::1 [root@easzlab-k8s-nfs-01 ~]# [root@easzlab-k8s-nfs-01 ~]# netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:33865 0.0.0.0:* LISTEN - tcp 0 0 0.0.0.0:6379 0.0.0.0:* LISTEN 16629/redis-server tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/init tcp 0 0 0.0.0.0:57011 0.0.0.0:* LISTEN 764/rpc.mountd tcp 0 0 0.0.0.0:57299 0.0.0.0:* LISTEN 764/rpc.mountd tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 677/systemd-resolve tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 826/sshd: /usr/sbin tcp 0 0 127.0.0.1:6010 0.0.0.0:* LISTEN 15724/sshd: root@pt tcp 0 0 0.0.0.0:34651 0.0.0.0:* LISTEN 764/rpc.mountd tcp 0 0 0.0.0.0:2049 0.0.0.0:* LISTEN - tcp6 0 0 :::35459 :::* LISTEN 764/rpc.mountd tcp6 0 0 ::1:6379 :::* LISTEN 16629/redis-server tcp6 0 0 :::111 :::* LISTEN 1/init tcp6 0 0 :::46803 :::* LISTEN - tcp6 0 0 :::22 :::* LISTEN 826/sshd: /usr/sbin tcp6 0 0 :::58807 :::* LISTEN 764/rpc.mountd tcp6 0 0 ::1:6010 :::* LISTEN 15724/sshd: root@pt tcp6 0 0 :::55357 :::* LISTEN 764/rpc.mountd tcp6 0 0 :::2049 :::* LISTEN - [root@easzlab-k8s-nfs-01 ~]#
#k8s集群安装Endpoints [root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 0.3.Endpoints.yaml service/mysql-production-server-name created endpoints/mysql-production-server-name created [root@easzlab-deploy 4.Ingress-cases]# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.100.0.1 <none> 443/TCP 14d my-external-test-name ExternalName <none> www.cyhsre.com <none> 17m mysql-production-server-name ClusterIP 10.100.8.102 <none> 6379/TCP 4s [root@easzlab-deploy 4.Ingress-cases]# kubectl get ep NAME ENDPOINTS AGE kubernetes 172.16.88.157:6443,172.16.88.158:6443,172.16.88.159:6443 14d mysql-production-server-name 172.16.88.169:6379 18s [root@easzlab-deploy 4.Ingress-cases]#
Service 如果是 cluster 类型那么从 clusterIP 到 pod 是默认是 TCP 协议，TCP 支持MySQL、Redis 等特定服务，另外还有 UDP 和 SCTP 协议。
二、ingress简介
Ingress： https://kubernetes.io/zh/docs/concepts/services-networking/ingress/
- Ingress 是 kubernetes API 中的标准资源类型之一， ingress 实现的功能是在应用层对客户端请求的 host 名称或请求的 URL 路径把请求转发到指定的 service 资源的规则， 即用于将 kubernetes 集群外部的请求资源转发之集群内部的 service， 再被 service 转发之 pod处理客户端的请求。
Ingress controller ：https://kubernetes.io/zh/docs/concepts/services-networking/ingress-controllers/
- Ingress 资源需要指定监听地址、 请求的 host 和 URL 等配置， 然后根据这些规则的匹配机制将客户端的请求进行转发， 这种能够为 ingress 配置资源监听并转发流量的组件称为ingress 控制器(ingress controller)， ingress controller 是 kubernetes 的一个附件， 类似于dashboard 或者 flannel 一样， 需要单独部署。
Ingress 选型：https://kubernetes.io/zh/docs/concepts/services-networking/ingress-controllers/
部署 ingress controller：https://kubernetes.github.io/ingress-nginx/deploy/
NodePort 方式：https://kubernetes.github.io/ingress-nginx/deploy/#bare-metal
官方配置文档：https://kubernetes.io/zh/docs/concepts/services-networking/ingress/
三、部署 web 服务及 controller
安装部署：
- https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/index.md
- https://docs.nginx.com/nginx-ingress-controller/intro/overview/
- https://kubernetes.io/docs/concepts/services-networking/ingress/
- https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/
ingress 项目
3.1、部署web服务
[root@easzlab-deploy 4.Ingress-cases]# cat tomcat-app1.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
labels:
app: magedu-tomcat-app1-deployment-label
name: magedu-tomcat-app1-deployment
namespace: magedu
spec:
replicas: 1
selector:
matchLabels:
app: magedu-tomcat-app1-selector
template:
metadata:
labels:
app: magedu-tomcat-app1-selector
spec:
containers:
- name: magedu-tomcat-app1-container
image: tomcat:7.0.94-alpine
#command: ["/apps/tomcat/bin/run_tomcat.sh"]
#imagePullPolicy: IfNotPresent
imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: TCP
name: http
env:
- name: "password"
value: "123456"
- name: "age"
value: "18"
resources:
limits:
cpu: 1
memory: "512Mi"
requests:
cpu: 500m
memory: "512Mi"
---
kind: Service
apiVersion: v1
metadata:
labels:
app: magedu-tomcat-app1-service-label
name: magedu-tomcat-app1-service
namespace: magedu
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
nodePort: 40003
selector:
app: magedu-tomcat-app1-selector
[root@easzlab-deploy 4.Ingress-cases]#
[root@easzlab-deploy 4.Ingress-cases]#
[root@easzlab-deploy 4.Ingress-cases]# cat tomcat-app2.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
labels:
app: magedu-tomcat-app2-deployment-label
name: magedu-tomcat-app2-deployment
namespace: magedu
spec:
replicas: 1
selector:
matchLabels:
app: magedu-tomcat-app2-selector
template:
metadata:
labels:
app: magedu-tomcat-app2-selector
spec:
containers:
- name: magedu-tomcat-app2-container
image: tomcat:7.0.94-alpine
#command: ["/apps/tomcat/bin/run_tomcat.sh"]
#imagePullPolicy: IfNotPresent
imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: TCP
name: http
env:
- name: "password"
value: "123456"
- name: "age"
value: "18"
resources:
limits:
cpu: 1
memory: "512Mi"
requests:
cpu: 500m
memory: "512Mi"
---
kind: Service
apiVersion: v1
metadata:
labels:
app: magedu-tomcat-app2-service-label
name: magedu-tomcat-app2-service
namespace: magedu
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
nodePort: 40004
selector:
app: magedu-tomcat-app2-selector
[root@easzlab-deploy 4.Ingress-cases]#
准备测试页面
[root@easzlab-deploy 4.Ingress-cases]# kubectl get pod -n magedu NAME READY STATUS RESTARTS AGE magedu-jenkins-deployment-79bbd88cb7-84999 1/1 Running 0 9d magedu-jenkins-deployment-79bbd88cb7-pmd9z 1/1 Running 0 7d8h magedu-tomcat-app1-deployment-5fd7d8767f-v7hbl 1/1 Running 0 94m magedu-tomcat-app2-deployment-65c89fc96f-p8mnd 1/1 Running 0 94m [root@easzlab-deploy 4.Ingress-cases]# kubectl exec -it -n magedu magedu-tomcat-app1-deployment-5fd7d8767f-v7hbl bash kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead. bash-4.4# cd webapps/ bash-4.4# mkdir app1 bash-4.4# echo "tomcat-app1-ingress-nginx-test" > app1/index.jsp bash-4.4# exit exit [root@easzlab-deploy 4.Ingress-cases]# kubectl exec -it -n magedu magedu-tomcat-app2-deployment-65c89fc96f-p8mnd bash kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead. bash-4.4# cd webapps/ bash-4.4# mkdir app2 bash-4.4# echo "tomcat-app2-ingress-nginx-test" > app2/index.jsp bash-4.4# bash-4.4# exit exit [root@easzlab-deploy 4.Ingress-cases]#
3.2、部署ingress
cat 1.ingress-nginx-controller-v1.3.0_deployment.yaml
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
---
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resourceNames:
- ingress-controller-leader
resources:
- configmaps
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- coordination.k8s.io
resourceNames:
- ingress-controller-leader
resources:
- leases
verbs:
- get
- update
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
verbs:
- list
- watch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: v1
data:
allow-snippet-annotations: "true"
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-controller
namespace: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- appProtocol: http
name: http
port: 80
protocol: TCP
targetPort: http
nodePort: 50080
- appProtocol: http #kubernetes v1.20 stable,appProtocol字段提供了一种为每个Service端口指定应用协议的方式,此字段的取值会被映射到对应的Endpoints
name: prometheus-metrics-port
port: 10254
protocol: TCP
targetPort: 10254 #ingress-nginx-controller内置的指标数据采集端口
nodePort: 61254
# - name: metrics-port
# port: 10254
# targetPort: 10254
# nodePort: 50254
# protocol: TCP
- appProtocol: https
name: https
port: 443
protocol: TCP
targetPort: https
nodePort: 50443
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: NodePort
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
ports:
- appProtocol: https
name: https-webhook
port: 443
targetPort: webhook
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
minReadySeconds: 0
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
spec:
containers:
- args:
- /nginx-ingress-controller
- --election-id=ingress-controller-leader
- --controller-class=k8s.io/ingress-nginx
- --ingress-class=nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/ingress-nginx-controller:v1.3.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
runAsUser: 101
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 300
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission-create
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission-create
spec:
containers:
- args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/kube-webhook-certgen:v1.3.0
imagePullPolicy: IfNotPresent
name: create
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission-patch
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission-patch
spec:
containers:
- args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/kube-webhook-certgen:v1.3.0
imagePullPolicy: IfNotPresent
name: patch
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: nginx
spec:
controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.3.0
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None
View Code
3.3、实现单 host 及多 host 的 ingress
基于客户端请求的 host 域名进行转发
3.3.1、实现单个虚拟主机
[root@easzlab-deploy 4.Ingress-cases]# cat 2.1.ingress_single-host.yaml
#apiVersion: networking.k8s.io/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-web
namespace: magedu
annotations:
kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
nginx.ingress.kubernetes.io/use-regex: "true" ##指定后面rules定义的path可以使用正则表达式
nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" ##连接超时时间,默认为5s
nginx.ingress.kubernetes.io/proxy-send-timeout: "600" ##后端服务器回转数据超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-read-timeout: "600" ##后端服务器响应超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-body-size: "50m" ##客户端上传文件，最大大小，默认为20m
#nginx.ingress.kubernetes.io/rewrite-target: / ##URL重写
nginx.ingress.kubernetes.io/app-root: /index.html
#spec:
# rules: #路由规则
# - host: www.jiege.com ##客户端访问的host域名
# http:
# paths:
# - path:
# backend:
# serviceName: magedu-nginx-service #转发至哪个service
# servicePort: 80 ##转发至service的端口号
spec:
rules:
- host: www.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app1-service
port:
number: 80
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 2.1.ingress_single-host.yaml
ingress.networking.k8s.io/nginx-web created
[root@easzlab-deploy 4.Ingress-cases]#
配置负载均衡
[root@easzlab-haproxy-keepalive-01 ~]# cat /etc/haproxy/haproxy.cfg
................
listen tomcat-ingress-80
bind 172.16.88.200:80
mode tcp
server easzlab-k8s-master-01 172.16.88.157:50080 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-02 172.16.88.158:50080 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-03 172.16.88.159:50080 check inter 2000 fall 3 rise 5
[root@easzlab-haproxy-keepalive-01 ~]#
[root@easzlab-haproxy-keepalive-01 ~]# systemctl restart haproxy
添加本地hosts解析
访问tomcat-app2失败，因为app2没加入匹配规则
3.3.2、实现多个虚拟主机
[root@easzlab-deploy 4.Ingress-cases]# vi 2.2.ingress_multi-host.yaml
[root@easzlab-deploy 4.Ingress-cases]# cat 2.2.ingress_multi-host.yaml
#apiVersion: networking.k8s.io/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-web
namespace: magedu
annotations:
kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
nginx.ingress.kubernetes.io/use-regex: "true" ##指定后面rules定义的path可以使用正则表达式
nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" ##连接超时时间,默认为5s
nginx.ingress.kubernetes.io/proxy-send-timeout: "600" ##后端服务器回转数据超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-read-timeout: "600" ##后端服务器响应超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-body-size: "10m" ##客户端上传文件，最大大小，默认为20m
#nginx.ingress.kubernetes.io/rewrite-target: / ##URL重写
nginx.ingress.kubernetes.io/app-root: /index.html
#spec:
# rules:
# - host: www.jiege.com
# http:
# paths:
# - path:
# backend:
# serviceName: magedu-tomcat-app1-service
# servicePort: 80
# - host: mobile.jiege.com
# http:
# paths:
# - path:
# backend:
# serviceName: magedu-tomcat-app2-service
# servicePort: 80
spec:
rules:
- host: www.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app1-service
port:
number: 80
- host: mobile.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app2-service
port:
number: 80
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 2.2.ingress_multi-host.yaml
ingress.networking.k8s.io/nginx-web created
[root@easzlab-deploy 4.Ingress-cases]#
3.4、实现基于 URL 的 ingress
基于客户端请求的同一个 host 不同的 URL 进行转发
[root@easzlab-deploy 4.Ingress-cases]# vi 3.1.ingress-url.yaml
[root@easzlab-deploy 4.Ingress-cases]# cat 3.1.ingress-url.yaml
#apiVersion: networking.k8s.io/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-web
namespace: magedu
annotations:
kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
nginx.ingress.kubernetes.io/use-regex: "true" ##指定后面rules定义的path可以使用正则表达式
nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" ##连接超时时间,默认为5s
nginx.ingress.kubernetes.io/proxy-send-timeout: "600" ##后端服务器回转数据超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-read-timeout: "600" ##后端服务器响应超时时间,默认为60s
nginx.ingress.kubernetes.io/proxy-body-size: "10m" ##客户端上传文件，最大大小，默认为20m
#nginx.ingress.kubernetes.io/rewrite-target: / ##URL重写
nginx.ingress.kubernetes.io/app-root: /index.html
spec:
rules:
# - host: www.jiege.com
# http:
# paths:
# - path: /magedu
# backend:
# serviceName: magedu-tomcat-app1-service
# servicePort: 80
# - path: /magedu2
# backend:
# serviceName: magedu-tomcat-app2-service
# servicePort: 80
- host: www.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/app1"
backend:
service:
name: magedu-tomcat-app1-service
port:
number: 80
- pathType: Prefix
path: "/app2"
backend:
service:
name: magedu-tomcat-app2-service
port:
number: 80
[root@easzlab-deploy 4.Ingress-cases]# kubectl delete -f 2.2.ingress_multi-host.yaml
ingress.networking.k8s.io "nginx-web" deleted
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 3.1.ingress-url.yaml
ingress.networking.k8s.io/nginx-web created
[root@easzlab-deploy 4.Ingress-cases]#
3.5、ingress 实现单域名及多域名 https
3.5.1、ingress单域名https访问
[root@easzlab-deploy 4.Ingress-cases]# cat 4.1.ingress-https-magedu_single-host.yaml
#apiVersion: networking.k8s.io/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-web
namespace: magedu
annotations:
kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
nginx.ingress.kubernetes.io/ssl-redirect: 'true' #SSL重定向，即将http请求强制重定向至https，等于nginx中的全站https
spec:
tls:
- hosts:
- www.cyhsre.com
secretName: tls-secret-www
# rules:
# - host: www.jiege.com
# http:
# paths:
# - path: /
# backend:
# serviceName: magedu-tomcat-app1-service
# servicePort: 80
rules:
- host: www.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app1-service
port:
number: 80
[root@easzlab-deploy 4.Ingress-cases]# kubectl delete -f 3.1.ingress-url.yaml
ingress.networking.k8s.io "nginx-web" deleted
[root@easzlab-deploy 4.Ingress-cases]#
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 4.1.ingress-https-magedu_single-host.yaml
ingress.networking.k8s.io/nginx-web created
[root@easzlab-deploy 4.Ingress-cases]#
证书签发
mkdir certs cd certs openssl req -x509 -sha256 -newkey rsa:4096 -keyout ca.key -out ca.crt -days 3560 -nodes -subj '/CN=www.cyhsre.com' openssl req -new -newkey rsa:4096 -keyout server.key -out server.csr -nodes -subj '/CN=www.cyhsre.com' openssl x509 -req -sha256 -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt kubectl create secret tls tls-secret-www --cert=./server.crt --key=./server.key -n magedu
配置负载均衡nodeport 443端口转发
[root@easzlab-haproxy-keepalive-01 ~]# vi /etc/haproxy/haproxy.cfg
listen tomcat-ingress-443
bind 172.16.88.200:443
mode tcp
server easzlab-k8s-master-01 172.16.88.157:50443 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-02 172.16.88.158:50443 check inter 2000 fall 3 rise 5
server easzlab-k8s-master-03 172.16.88.159:50443 check inter 2000 fall 3 rise 5
[root@easzlab-haproxy-keepalive-01 ~]# systemctl restart haproxy
3.5.2、ingress 多域名https访问
创建mobile证书
mkdir mobile cd mobile openssl req -x509 -sha256 -newkey rsa:4096 -keyout ca.key -out ca.crt -days 3560 -nodes -subj '/CN=mobile.cyhsre.com' openssl req -new -newkey rsa:4096 -keyout server.key -out server.csr -nodes -subj '/CN=mobile.cyhsre.com' openssl x509 -req -sha256 -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt kubectl create secret tls tls-secret-mobile --cert=./server.crt --key=./server.key -n magedu
[root@easzlab-deploy 4.Ingress-cases]# cat 4.2.ingress-https-magedu_multi-host.yaml
#apiVersion: networking.k8s.io/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-web-mobile
namespace: magedu
annotations:
kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
nginx.ingress.kubernetes.io/ssl-redirect: 'true'
spec:
tls:
- hosts:
- mobile.cyhsre.com
secretName: tls-secret-mobile
- hosts:
- www.jiege.com
secretName: tls-secret-www
rules:
- host: www.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app1-service
port:
number: 80
- host: mobile.cyhsre.com
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: magedu-tomcat-app2-service
port:
number: 80
[root@easzlab-deploy 4.Ingress-cases]#
[root@easzlab-deploy 4.Ingress-cases]# kubectl apply -f 4.2.ingress-https-magedu_multi-host.yaml
ingress.networking.k8s.io/nginx-web-mobile created
[root@easzlab-deploy 4.Ingress-cases]#
