diff --git a/README.md b/README.md index ce3191a..8e4a0b4 100644 --- a/README.md +++ b/README.md @@ -372,6 +372,63 @@ To enable [Ingress NGINX](https://github.com/kubernetes/ingress-nginx), an Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer, it is sufficient to declare: +```yaml +k8s_ingress_nginx_enable: true +``` + +This will install the Ingress NGINX controller that can be used for different +purposes. + +#### Ingress NGINX on control-planes + +For example it is possible to use Ingress NGINX by exposing the `80` and `443` +ports on the balanced IP managed by haproxy, by declaring this: + +```yaml +k8s_ingress_nginx_enable: true +k8s_ingress_nginx_haproxy_conf: true +k8s_ingress_nginx_services: + - name: ingress-nginx-externalip + spec: + externalIPs: + - 192.168.122.199 + ports: + - name: port-1 + port: 80 + protocol: TCP + - name: port-2 + port: 443 + protocol: TCP + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx +``` + +This will expose both ports on the balanced IP (in this case `192.168.122.199` +and will make the service responding there. + +To test it just try this: + +```console +$ kubectl create deployment demo --image=httpd --port=80 +deployment.apps/demo created + +$ kubectl expose deployment demo +service/demo exposed + +$ kubectl create ingress demo --class=nginx --rule="demo.192.168.122.199.nip.io/*=demo:80" +ingress.networking.k8s.io/demo created + +$ curl http://demo.192.168.122.199.nip.io +

It works!

+``` + +#### Ingress NGINX with MetalLB + +Another way is to use it in combination with MetalLB, by declaring a +`LoadBalancer` service, as follows: + ```yaml k8s_ingress_nginx_enable: true k8s_ingress_nginx_services: diff --git a/defaults/main.yml b/defaults/main.yml index cb4e3a3..7800bc2 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -112,6 +112,7 @@ k8s_metallb_ports: k8s_metallb_pools: {} k8s_ingress_nginx_enable: false +k8s_ingress_nginx_haproxy_conf: false k8s_ingress_nginx_namespace: ingress-nginx k8s_ingress_nginx_services: {} diff --git a/templates/ingress-nginx/ingress-nginx-deployment.yaml.j2 b/templates/ingress-nginx/ingress-nginx-deployment.yaml.j2 index 3fa41b2..4f60a41 100644 --- a/templates/ingress-nginx/ingress-nginx-deployment.yaml.j2 +++ b/templates/ingress-nginx/ingress-nginx-deployment.yaml.j2 @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx namespace: {{ k8s_ingress_nginx_namespace }} --- @@ -27,7 +27,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission namespace: {{ k8s_ingress_nginx_namespace }} --- @@ -39,7 +39,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx namespace: {{ k8s_ingress_nginx_namespace }} rules: @@ -90,25 +90,10 @@ rules: - get - list - watch -- apiGroups: - - "" - resourceNames: - - ingress-controller-leader - resources: - - configmaps - verbs: - - get - - update -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - apiGroups: - coordination.k8s.io resourceNames: - - ingress-controller-leader + - ingress-nginx-leader resources: - leases verbs: @@ -144,7 +129,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission namespace: {{ k8s_ingress_nginx_namespace }} rules: @@ -163,7 +148,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx rules: - apiGroups: @@ -245,7 +230,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission rules: - apiGroups: @@ -264,7 +249,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx namespace: {{ k8s_ingress_nginx_namespace }} roleRef: @@ -284,7 +269,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission namespace: {{ k8s_ingress_nginx_namespace }} roleRef: @@ -303,7 +288,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io @@ -322,7 +307,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission roleRef: apiGroup: rbac.authorization.k8s.io @@ -335,7 +320,7 @@ subjects: --- apiVersion: v1 data: - allow-snippet-annotations: "true" + allow-snippet-annotations: "false" kind: ConfigMap metadata: labels: @@ -343,7 +328,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-controller namespace: {{ k8s_ingress_nginx_namespace }} --- @@ -355,10 +340,11 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-controller namespace: {{ k8s_ingress_nginx_namespace }} spec: + externalTrafficPolicy: Local ipFamilies: - IPv4 ipFamilyPolicy: SingleStack @@ -377,7 +363,7 @@ spec: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - type: NodePort + type: LoadBalancer --- apiVersion: v1 kind: Service @@ -387,7 +373,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-controller-admission namespace: {{ k8s_ingress_nginx_namespace }} spec: @@ -410,7 +396,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-controller namespace: {{ k8s_ingress_nginx_namespace }} spec: @@ -421,23 +407,31 @@ spec: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.10.0 spec: containers: - args: - /nginx-ingress-controller - - --election-id=ingress-controller-leader + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-nginx-leader - --controller-class=k8s.io/ingress-nginx - --ingress-class=nginx - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - --validating-webhook=:8443 - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key + - --enable-metrics=false env: - name: POD_NAME valueFrom: @@ -449,7 +443,7 @@ spec: fieldPath: metadata.namespace - name: LD_PRELOAD value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.4.0@sha256:34ee929b111ffc7aa426ffd409af44da48e5a0eea1eb2207994d9e0c0882d143 + image: registry.k8s.io/ingress-nginx/controller:v1.10.0@sha256:42b3f0e5d0846876b1791cd3afeb5f1cbbe4259d6f35651dcc1b5c980925379c imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -492,13 +486,17 @@ spec: cpu: 100m memory: 90Mi securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true runAsUser: 101 + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /usr/local/certificates/ name: webhook-cert @@ -521,7 +519,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission-create namespace: {{ k8s_ingress_nginx_namespace }} spec: @@ -532,7 +530,7 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission-create spec: containers: @@ -546,18 +544,22 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.0@sha256:44d1d0e9f19c63f58b380c5fddaca7cf22c7cee564adeff365225a5df5ef3334 imagePullPolicy: IfNotPresent name: create securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault nodeSelector: kubernetes.io/os: linux restartPolicy: OnFailure - securityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 serviceAccountName: ingress-nginx-admission --- apiVersion: batch/v1 @@ -568,7 +570,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission-patch namespace: {{ k8s_ingress_nginx_namespace }} spec: @@ -579,7 +581,7 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission-patch spec: containers: @@ -595,18 +597,22 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.0@sha256:44d1d0e9f19c63f58b380c5fddaca7cf22c7cee564adeff365225a5df5ef3334 imagePullPolicy: IfNotPresent name: patch securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault nodeSelector: kubernetes.io/os: linux restartPolicy: OnFailure - securityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 serviceAccountName: ingress-nginx-admission --- apiVersion: networking.k8s.io/v1 @@ -617,7 +623,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: nginx spec: controller: k8s.io/ingress-nginx @@ -630,9 +636,8 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.4.0 + app.kubernetes.io/version: 1.10.0 name: ingress-nginx-admission - namespace: {{ k8s_ingress_nginx_namespace }} webhooks: - admissionReviewVersions: - v1 diff --git a/templates/multi-control-plane/haproxy.cfg.j2 b/templates/multi-control-plane/haproxy.cfg.j2 index a058726..568b4d7 100644 --- a/templates/multi-control-plane/haproxy.cfg.j2 +++ b/templates/multi-control-plane/haproxy.cfg.j2 @@ -52,3 +52,51 @@ backend apiserver server control-plane-{{ loop.index }} {{ host }}:{{ k8s_control_plane_port }} check {% endif %} {% endfor %} + +{% if k8s_ingress_nginx_haproxy_conf | bool %} +#--------------------------------------------------------------------- +# http and https frontends which proxys to the ingress backends +#--------------------------------------------------------------------- +frontend ingresshttp + bind *:80 + mode tcp + option tcplog + default_backend ingresshttp + +frontend ingresshttps + bind *:443 + mode tcp + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend ingresshttps + +#--------------------------------------------------------------------- +# round robin balancing between the various k8s control planes +#--------------------------------------------------------------------- +backend ingresshttp + option httpchk GET /readyz HTTP/1.0 + option log-health-checks + http-check expect status 200 + mode tcp + balance roundrobin + default-server verify none check-ssl inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 5000 maxqueue 5000 weight 100 +{% for host in ansible_play_batch %} +{% if hostvars[host].k8s_role == 'control-plane' %} + server control-plane-{{ loop.index }} {{ host }}:80 check +{% endif %} +{% endfor %} + +backend ingresshttps + option httpchk GET /readyz HTTP/1.0 + option log-health-checks + http-check expect status 200 + mode tcp + balance roundrobin + default-server verify none check-ssl inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 5000 maxqueue 5000 weight 100 +{% for host in ansible_play_batch %} +{% if hostvars[host].k8s_role == 'control-plane' %} + server control-plane-{{ loop.index }} {{ host }}:443 check +{% endif %} +{% endfor %} + +{% endif %} diff --git a/tests/inventory/group_vars/kubelab.yml b/tests/inventory/group_vars/kubelab.yml index 3567e64..cab10cc 100644 --- a/tests/inventory/group_vars/kubelab.yml +++ b/tests/inventory/group_vars/kubelab.yml @@ -59,7 +59,23 @@ k8s_metallb_pools: addresses: '192.168.122.100-192.168.122.130' k8s_ingress_nginx_enable: true +k8s_ingress_nginx_haproxy_conf: true k8s_ingress_nginx_services: + - name: ingress-nginx-externalip + spec: + externalIPs: + - 192.168.122.199 + ports: + - name: port-1 + port: 80 + protocol: TCP + - name: port-2 + port: 443 + protocol: TCP + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx - name: ingress-nginx-lb spec: type: LoadBalancer @@ -76,7 +92,6 @@ k8s_ingress_nginx_services: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - k8s_cert_manager_enable: true k8s_cert_manager_issuers: - name: letsencrypt