Support leader election for Connect-Inject controller (hashicorp#903)

* Support leader election for Connect-Inject controller

- Leader election required creating a configmap that is owned by the
  instance of the controller that is the leader. This required RBAC to
perform CRUD operations on configmaps to the controller.
- Additionally the latest version of the controller runtime requires the
  controller to read "leases" from kubernetes and we now have to support
that for the connect-inject controller.
- Increase the number of replicas of the connect inject deployment to 2
  as we can now be HA.

.circleci/config.yml

@@ -143,7 +143,7 @@ jobs:
                     -kubecontext="kind-dc1" \
                     -secondary-kubecontext="kind-dc2" \
                     -debug-directory="$TEST_RESULTS/debug" \
-                    -consul-k8s-image=kschoche/consul-k8s-dev2:2021-4-9 # TODO: change once feature-tproxy consul-k8s branch is merged
+                    -consul-k8s-image=ashwinvenkatesh/consul-k8s@sha256:f66106414f9e6b05271bd2944bf0a01308a6b9405221d73e89f2d00159189d6e # TODO: change once feature-tproxy consul-k8s branch is merged
               then
                 echo "Tests in ${pkg} failed, aborting early"
                 exit_code=1

templates/connect-inject-clusterrole.yaml

@@ -16,6 +16,15 @@ rules:
   - "get"
   - "list"
   - "watch"
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - update
 {{- if .Values.global.enablePodSecurityPolicies }}
 - apiGroups: ["policy"]
   resources: ["podsecuritypolicies"]

templates/connect-inject-deployment.yaml

@@ -19,7 +19,7 @@ metadata:
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: {{ template "consul.name" . }}

templates/connect-inject-leader-election-role.yaml

@@ -0,0 +1,40 @@
+{{- if or (and (ne (.Values.connectInject.enabled | toString) "-") .Values.connectInject.enabled) (and (eq (.Values.connectInject.enabled | toString) "-") .Values.global.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "consul.fullname" . }}-connect-inject-leader-election
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+{{- end }}

templates/connect-inject-leader-election-rolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if or (and (ne (.Values.connectInject.enabled | toString) "-") .Values.connectInject.enabled) (and (eq (.Values.connectInject.enabled | toString) "-") .Values.global.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "consul.fullname" . }}-connect-inject-leader-election
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "consul.fullname" . }}-connect-inject-leader-election
+subjects:
+- kind: ServiceAccount
+  name: {{ template "consul.fullname" . }}-connect-injector-webhook-svc-account
+  namespace: {{ .Release.Namespace }}
+{{- end }}