Deploy ESO on EKS

infra/aws/terraform/prow-build-cluster/resources/external-secrets/README.md

@@ -0,0 +1,8 @@
+Please run this command to upgrade External Secret Secrets Operator:
+
+```
+helm template external-secrets \
+   external-secrets/external-secrets  \
+   -f=values \
+   -n=external-secrets > external-secrets.yaml
+```

infra/aws/terraform/prow-build-cluster/resources/external-secrets/configmaps.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: google-adc
+  namespace: default
+data:
+  adc.json: |
+    {
+      "type": "external_account",
+      "audience": "//iam.googleapis.com/projects/773781448124/locations/global/workloadIdentityPools/prow-eks/providers/oidc",
+      "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+      "token_url": "https://sts.googleapis.com/v1/token",
+      "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/kubernetes-external-secrets@k8s-infra-prow-build.iam.gserviceaccount.com:generateAccessToken",
+      "credential_source": {
+        "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+        "format": {
+          "type": "text"
+        }
+      }
+    }

infra/aws/terraform/prow-build-cluster/resources/external-secrets/values

@@ -0,0 +1,23 @@
+extraVolumes:
+  - name: google-iam-token
+    projected:
+      defaultMode: 420
+      sources:
+      - serviceAccountToken:
+          audience: sts.googleapis.com
+          expirationSeconds: 86400
+          path: token
+  - name: google-adc
+    configMap:
+      name: google-adc
+
+extraEnv:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /etc/google/adc.json
+
+extraVolumeMounts:
+  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+    name: google-iam-token
+    readOnly: true
+  - mountPath: /etc/google
+    name: google-adc

infra/aws/terraform/prow-build-cluster/resources/kube-system/create-loop-devs_daemonset.yaml

@@ -0,0 +1,54 @@
+# A daemonset to create /dev/loopX device nodes before they are
+# needed.
+#
+# Normally, new nodes are created dynamically by the kernel. But nodes
+# in a KIND cluster are started with a copy of /dev from the host and
+# loop devices created later on do not show up in that static /dev
+# (https://github.com/kubernetes-sigs/kind/issues/1248). Creating
+# "enough" (100 in this daemonset) in advance avoids running out of
+# loop devices.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: create-loop-devs
+  namespace: kube-system
+  labels:
+    app: create-loop-devs
+spec:
+  selector:
+    matchLabels:
+      name: create-loop-devs
+  template:
+    metadata:
+      labels:
+        name: create-loop-devs
+    spec:
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: loopdev
+        command:
+        - sh
+        - -c
+        - |
+          while true; do
+            for i in $(seq 0 1000); do
+                if ! [ -e /dev/loop$i ]; then
+                  mknod /dev/loop$i b 7 $i
+                fi
+            done
+            sleep 100000000
+          done
+        image: alpine:3.6
+        imagePullPolicy: IfNotPresent
+        resources: {}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: dev
+          mountPath: /dev
+      volumes:
+      - name: dev
+        hostPath:
+          path: /dev

infra/aws/terraform/prow-build-cluster/resources/kube-system/tune-sysctls_daemonset.yaml

@@ -0,0 +1,47 @@
+# a simple daemonset to tune sysctls
+# intended to be used in a prow build cluster
+# see https://github.com/kubernetes/test-infra/pull/13515 for more info
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: tune-sysctls
+  namespace: kube-system
+  labels:
+    app: tune-sysctls
+spec:
+  selector:
+    matchLabels:
+      name: tune-sysctls
+  template:
+    metadata:
+      labels:
+        name: tune-sysctls
+    spec:
+      hostNetwork: true
+      hostPID: true
+      hostIPC: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: setsysctls
+        command:
+        - sh
+        - -c
+        - |
+          while true; do
+            sysctl -w fs.inotify.max_user_watches=524288
+            sleep 10
+          done
+        image: alpine:3.6
+        imagePullPolicy: IfNotPresent
+        resources: {}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: sys
+          mountPath: /sys
+      volumes:
+      - name: sys
+        hostPath:
+          path: /sys

infra/aws/terraform/prow-build-cluster/resources/namespaces.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: k8s-infra-test-pods
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-pods
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-secrets

infra/aws/terraform/prow-build-cluster/resources/test-pods/build-serviceaccounts.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prow-build
+  namespace: test-pods

infra/aws/terraform/prow-build-cluster/resources/test-pods/cpu-limit-range_limitrange.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: cpu-limit-range
+  namespace: test-pods
+spec:
+  limits:
+  - defaultRequest:
+      cpu: 250m
+    type: Container

infra/aws/terraform/prow-build-cluster/resources/test-pods/externalsecrets.yaml

@@ -0,0 +1,81 @@
+# This is a place holder for adding kubernetes external secrets, please add the
+# ExternalSecret CR here, separated by `---`.
+
+# NOTE !!!
+# THIS CLUSTER USES EXTERNAL SECRETS OPERATOR INSTEAD OF KUBERNETES EXTERNAL SECRETS.
+# The CRDs are different from other prow clusters. Sample guide that explains the diff https://wiki.cac.washington.edu/display/MCI/Transition+from+KES+to+ESO
+# ---
+# apiVersion: kubernetes-client.io/v1
+# kind: ExternalSecret
+# metadata:
+#   name: service-account             # The name of the Kubernetes Secret
+#   namespace: test-pods
+# spec:
+#   backendType: gcpSecretsManager
+#   projectId: k8s-infra-prow-build
+#   data:
+#   - key: prow-build-service-account # The name of the GSM Secret
+#     name: service-account.json      # The key to write to in the Kubernetes Secret
+#     version: latest                 # The version of the GSM Secret
+# ---
+# apiVersion: kubernetes-client.io/v1
+# kind: ExternalSecret
+# metadata:
+#   name: ssh-key-secret                          # The name of the Kubernetes Secret
+#   namespace: test-pods
+# spec:
+#   backendType: gcpSecretsManager
+#   projectId: k8s-infra-prow-build
+#   data:
+#   - key: prow-build-ssh-key-secret-ssh-public   # The name of the GSM Secret
+#     name: ssh-public                            # The key to write to in the Kubernetes Secret
+#     version: latest                             # The version of the GSM Secret
+#   - key: prow-build-ssh-key-secret-ssh-private  # The name of the GSM Secret
+#     name: ssh-private                           # The key to write to in the Kubernetes Secret
+#     version: latest                             # The version of the GSM Secret
+# ---
+# apiVersion: kubernetes-client.io/v1
+# kind: ExternalSecret
+# metadata:
+#   name: aws-credentials-768319786644           # The name of the Kubernetes Secret
+#   namespace: test-pods
+# spec:
+#   backendType: gcpSecretsManager
+#   projectId: k8s-infra-prow-build
+#   data:
+#   - key: gke_k8s-prow-builds_us-central1-f_prow__test-pods__aws-credentials-768319786644   # The name of the GSM Secret
+#     name: credentials                                                                      # The key to write to in the Kubernetes Secret
+#     version: latest                                                                        # The version of the GSM Secret
+#     property: credentials
+# ---
+# apiVersion: kubernetes-client.io/v1
+# kind: ExternalSecret
+# metadata:
+#   name: aws-credentials-607362164682           # The name of the Kubernetes Secret
+#   namespace: test-pods
+# spec:
+#   backendType: gcpSecretsManager
+#   projectId: k8s-infra-prow-build
+#   data:
+#   - key: gke_k8s-prow-builds_us-central1-f_prow__test-pods__aws-credentials-607362164682   # The name of the GSM Secret
+#     name: credentials                                                                      # The key to write to in the Kubernetes Secret
+#     version: latest                                                                        # The version of the GSM Secret
+#     property: credentials
+# ---
+# apiVersion: kubernetes-client.io/v1
+# kind: ExternalSecret
+# metadata:
+#   name: aws-ssh-key-secret           # The name of the Kubernetes Secret
+#   namespace: test-pods
+# spec:
+#   backendType: gcpSecretsManager
+#   projectId: k8s-infra-prow-build
+#   data:
+#   - key: gke_k8s-prow-builds_us-central1-f_prow__test-pods__aws-ssh-key-secret   # The name of the GSM Secret
+#     name: aws-ssh-private                                                        # The key to write to in the Kubernetes Secret
+#     version: latest                                                              # The version of the GSM Secret
+#     property: aws-ssh-private
+#   - key: gke_k8s-prow-builds_us-central1-f_prow__test-pods__aws-ssh-key-secret   # The name of the GSM Secret
+#     name: aws-ssh-public                                                         # The key to write to in the Kubernetes Secret
+#     version: latest                                                              # The version of the GSM Secret
+#     property: aws-ssh-public

infra/aws/terraform/prow-build-cluster/resources/test-pods/mem-limit-range_limitrange.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: mem-limit-range
+  namespace: test-pods
+spec:
+  limits:
+  - defaultRequest:
+      memory: 1Gi
+    type: Container

infra/aws/terraform/prow-build-cluster/resources/test-pods/test-pods-poddisruptionbudget.yaml

@@ -0,0 +1,31 @@
+# Copyright 2020 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The purpose of the PodDisruptionBudget here is to never allow evicting pods created by prow.
+# Eviction of pods can happen for one of two reasons:
+# * Cluster autoscaler downscaling
+# * Someome/Something using `kubectl drain`
+#
+# It is still possible to delete the pods via a normal delete call. See https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#the-eviction-api
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: prow-pods
+  namespace: test-pods
+spec:
+  maxUnavailable: 0
+  selector:
+    matchLabels:
+      created-by-prow: "true"

infra/gcp/terraform/k8s-infra-prow-build/serviceaccounts.tf

@@ -26,16 +26,38 @@ locals {
       description       = "sync K8s secrets from GSM in this and other projects"
       project_roles     = ["roles/secretmanager.secretAccessor"],
       cluster_namespace = "kubernetes-external-secrets"
+      additional_workload_identity_principals = [
+        "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool_provider.eks_cluster.name}/attribute.sub/system:serviceaccount:default:default"
+      ]
     }
   }
 }
 
+
+resource "google_iam_workload_identity_pool" "eks_cluster" {
+  workload_identity_pool_id = "prow-eks"
+  display_name              = "EKS Prow Cluster"
+}
+
+resource "google_iam_workload_identity_pool_provider" "eks_cluster" {
+  workload_identity_pool_id          = google_iam_workload_identity_pool.eks_cluster.workload_identity_pool_id
+  workload_identity_pool_provider_id = "oidc"
+  attribute_mapping = {
+    "google.subject" = "assertion.sub"
+  }
+  oidc {
+    issuer_uri        = "https://oidc.eks.us-east-2.amazonaws.com/id/ASK MARKO"
+    allowed_audiences = ["sts.googleapis.com"]
+  }
+}
+
 module "workload_identity_service_accounts" {
-  for_each          = local.workload_identity_service_accounts
-  source            = "../modules/workload-identity-service-account"
-  project_id        = module.project.project_id
-  name              = each.key
-  description       = each.value.description
-  cluster_namespace = lookup(each.value, "cluster_namespace", local.pod_namespace)
-  project_roles     = lookup(each.value, "project_roles", [])
+  for_each                                = local.workload_identity_service_accounts
+  source                                  = "../modules/workload-identity-service-account"
+  project_id                              = module.project.project_id
+  name                                    = each.key
+  description                             = each.value.description
+  cluster_namespace                       = lookup(each.value, "cluster_namespace", local.pod_namespace)
+  project_roles                           = lookup(each.value, "project_roles", [])
+  additional_workload_identity_principals = lookup(each.value, "additional_workload_identity_principals", [])
 }

infra/gcp/terraform/modules/workload-identity-service-account/main.tf

@@ -33,7 +33,9 @@ resource "google_service_account" "serviceaccount" {
 data "google_iam_policy" "workload_identity" {
   binding {
     members = [
-      "serviceAccount:${local.cluster_project_id}.svc.id.goog[${var.cluster_namespace}/${local.cluster_serviceaccount_name}]",
+      concat("serviceAccount:${local.cluster_project_id}.svc.id.goog[${var.cluster_namespace}/${local.cluster_serviceaccount_name}]",
+        var.additional_workload_identity_principals
+      )
     ]
     role = "roles/iam.workloadIdentityUser"
   }

infra/gcp/terraform/modules/workload-identity-service-account/variables.tf

@@ -52,3 +52,10 @@ variable "project_roles" {
   type        = list(string)
   default     = []
 }
+
+
+variable "additional_workload_identity_principals" {
+  description = "A list of extra principals to grant WorkloadIdentityUser on the service account"
+  type        = list(string)
+  default     = []
+}