Update to ingress-nginx v0.44.0

The newer version will gracefully close the connection to the cr-syncer
when the config reloads, so this goes some way towards fixing #64
(although the cr-syncer should handled dropped connections better).

The Deployment is based on upstream's deploy.yaml, but without RBAC
policies, the namespace and the service account, to avoid changing too
much. The newer Deployment drops privileges and has more graceful
shutdown behavior.
https://github.com/kubernetes/ingress-nginx/blob/master/deploy/static/provider/cloud/deploy.yaml

This requires adjusting the ingress due to the following breaking
changes (which, incidentally, despite the "breaking changes" category in
the changelog, were not listed as breaking...):

- the secure-backends annotation has been removed
- path rewriting now uses regex groups rather than prefixes

While we're here, we can stop using the deprecated extensions/v1beta1
Ingress.

Change-Id: I24c266f21e489a5c9079e9f884699c84d6be6a0e

docs/developers/debug-auth.md

@@ -118,12 +118,12 @@ spec:
     app: debug
   type: ClusterIP
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   annotations:
     kubernetes.io/ingress.class: nginx
-    nginx.ingress.kubernetes.io/secure-backends: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: HTTP
   name: debug
 spec:
   rules:

docs/how-to/deploying-grpc-service.md

@@ -144,14 +144,13 @@ In contrast to the other tutorial, the Ingress tells nginx to forward incoming r
 
 [embedmd]:# (examples/greeter-service/greeter-server.yaml.tmpl yaml /^/ /---/)
 ```yaml
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: greeter-server-ingress
   annotations:
     kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/grpc-backend: "true"
-    nginx.ingress.kubernetes.io/secure-backends: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: GRPC
     nginx.ingress.kubernetes.io/auth-url: "http://token-vendor.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify?robots=true"
 spec:
   rules:

docs/how-to/deploying-service.md

@@ -162,7 +162,7 @@ Create a file called `hello-server.yaml` with the following contents:
 
 [embedmd]:# (examples/hello-service/server/hello-server.yaml yaml)
 ```yaml
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: hello-server-ingress

docs/how-to/examples/greeter-service/greeter-server.yaml.tmpl

@@ -1,11 +1,10 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: greeter-server-ingress
   annotations:
     kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/grpc-backend: "true"
-    nginx.ingress.kubernetes.io/secure-backends: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: GRPC
     nginx.ingress.kubernetes.io/auth-url: "http://token-vendor.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify?robots=true"
 spec:
   rules:

docs/how-to/examples/hello-service/server/hello-server.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: hello-server-ingress

src/app_charts/base/cloud/kubernetes-api.yaml

@@ -1,10 +1,10 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: kubernetes-api
   annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: "/"
-    nginx.ingress.kubernetes.io/secure-backends: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"  # seconds
     kubernetes.io/ingress.class: "nginx"
     # The web client will reach us through the oauth2-proxy and we need
@@ -29,7 +29,7 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /apis/core.kubernetes/
+      - path: /apis/core.kubernetes($|/)(.*)
         backend:
           serviceName: kubernetes
           servicePort: 443

src/app_charts/base/cloud/nginx-ingress-controller.yaml

@@ -1,11 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-ingress-controller
+data:
+  # The token-vendor checks the Original-URI header to accept tokens from query
+  # parameters.
+  proxy-add-original-uri-header: "true"
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: nginx-ingress-controller
   labels:
     k8s-app: nginx-ingress-controller
 spec:
-  replicas: 1
   selector:
     matchLabels:
       k8s-app: nginx-ingress-controller
@@ -14,38 +22,68 @@ spec:
       labels:
         k8s-app: nginx-ingress-controller
     spec:
-      terminationGracePeriodSeconds: 60
+      dnsPolicy: ClusterFirst
       containers:
-      - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.20.0
-        name: nginx-ingress-controller
-        readinessProbe:
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-          initialDelaySeconds: 10
-        ports:
-        - containerPort: 80
-          hostPort: 80
-        - containerPort: 443
-          hostPort: 443
-        env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-        args:
-        - /nginx-ingress-controller
-        - --v=3
-        - --default-backend-service=kube-system/default-http-backend
-        - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb
-        - --default-ssl-certificate=default/tls
+        - name: nginx-ingress-controller
+          image: k8s.gcr.io/ingress-nginx/controller:v0.44.0
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /wait-shutdown
+          args:
+            - /nginx-ingress-controller
+            - --v=3
+            - --default-backend-service=kube-system/default-http-backend
+            - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb
+            - --election-id=ingress-controller-leader
+            - --ingress-class=nginx
+            - --configmap=$(POD_NAMESPACE)/nginx-ingress-controller
+            - --default-ssl-certificate=default/tls
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+            runAsUser: 101
+            allowPrivilegeEscalation: true
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: LD_PRELOAD
+              value: /usr/local/lib/libmimalloc.so
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          ports:
+            - name: http
+              containerPort: 80
+            - name: https
+              containerPort: 443
+      nodeSelector:
+        kubernetes.io/os: linux
+      terminationGracePeriodSeconds: 300

src/app_charts/base/cloud/oauth2-proxy.yaml

@@ -51,12 +51,12 @@ spec:
     app: oauth2-proxy
   type: ClusterIP
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: oauth2-proxy
   annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: "/apis"
+    nginx.ingress.kubernetes.io/rewrite-target: /apis/$2
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"  # seconds
     kubernetes.io/ingress.class: "nginx"
 spec:
@@ -67,12 +67,12 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /web-apis
+      - path: /web-apis($|/)(.*)
         backend:
           serviceName: oauth2-proxy
           servicePort: http
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: oauth2-proxy-interactive

src/app_charts/base/cloud/token-vendor.yaml

@@ -44,7 +44,7 @@ spec:
         runAsUser: 65532
         runAsGroup: 65532
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: token-vendor
@@ -67,7 +67,7 @@ spec:
           serviceName: token-vendor
           servicePort: token-vendor
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: public-key-manager
@@ -87,7 +87,7 @@ spec:
           serviceName: token-vendor
           servicePort: token-vendor
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: public-key-access

src/app_charts/k8s-relay/cloud/ingress.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: kubernetes-relay-client
@@ -10,7 +10,7 @@ metadata:
     # for. This is important for requests like `kubectl logs -f` where the logs
     # may be silent for some time.
     nginx.ingress.kubernetes.io/proxy-read-timeout: "86400"
-    nginx.ingress.kubernetes.io/rewrite-target: /client
+    nginx.ingress.kubernetes.io/rewrite-target: /client/$2
 spec:
   tls:
   - hosts:
@@ -19,19 +19,19 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /apis/core.kubernetes-relay/client
+      - path: /apis/core.kubernetes-relay/client($|/)(.*)
         backend:
           serviceName: kubernetes-relay-server
           servicePort: 80
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: kubernetes-relay-server
   annotations:
     kubernetes.io/ingress.class: "nginx"
     ingress.kubernetes.io/proxy-body-size: "50m"
-    nginx.ingress.kubernetes.io/rewrite-target: /server
+    nginx.ingress.kubernetes.io/rewrite-target: /server/$2
     nginx.ingress.kubernetes.io/auth-url: "http://token-vendor.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify?robots=true"
 spec:
   tls:
@@ -41,7 +41,7 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /apis/core.kubernetes-relay/server
+      - path: /apis/core.kubernetes-relay/server($|/)(.*)
         backend:
           serviceName: kubernetes-relay-server
           servicePort: 80

src/app_charts/map/cloud/ingress.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: map-openapi
@@ -18,14 +18,13 @@ spec:
           serviceName: map-endpoint
           servicePort: 80
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: map-grpc
   annotations:
     kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/grpc-backend: "true"
-    nginx.ingress.kubernetes.io/secure-backends: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: GRPC
     nginx.ingress.kubernetes.io/auth-url: "http://token-vendor.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify?robots=true"
 spec:
   rules:

src/app_charts/prometheus/cloud/grafana-ingress.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: grafana
@@ -8,8 +8,8 @@ metadata:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/auth-url: "http://oauth2-proxy.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify"
     nginx.ingress.kubernetes.io/auth-signin: "https://{{ .Values.domain }}/oauth2/start?rd=$escaped_request_uri"
-    nginx.ingress.kubernetes.io/secure-backends: "false"
-    nginx.ingress.kubernetes.io/rewrite-target: "/"
+    nginx.ingress.kubernetes.io/backend-protocol: HTTP
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
     # HACK: oauth2-proxy will return 403, but nginx-ingress-controller only handles
     # 401 with an error page.
     nginx.ingress.kubernetes.io/configuration-snippet: |
@@ -22,7 +22,7 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /grafana
+      - path: /grafana($|/)(.*)
         backend:
           serviceName: prom-grafana
           servicePort: 80

src/app_charts/prometheus/cloud/prometheus-ingress.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: prometheus
@@ -8,8 +8,8 @@ metadata:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/auth-url: "http://oauth2-proxy.default.svc.cluster.local/apis/core.token-vendor/v1/token.verify"
     nginx.ingress.kubernetes.io/auth-signin: "https://{{ .Values.domain }}/oauth2/start?rd=$escaped_request_uri"
-    nginx.ingress.kubernetes.io/secure-backends: "false"
-    nginx.ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/backend-protocol: HTTP
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
     nginx.ingress.kubernetes.io/configuration-snippet: |
       error_page 403 = /oauth2/start?rd=$escaped_request_uri;
 spec:
@@ -20,7 +20,7 @@ spec:
   - host: {{ .Values.domain }}
     http:
       paths:
-      - path: /prometheus
+      - path: /prometheus($|/)(.*)
         backend:
           serviceName: kube-prometheus
           servicePort: 9090