Reconcile all the infra changes for deploying Prow (#7205)

* create prow bucket * deploy IAM changes to AWS/GCP to allow workload identity access to the clusters * use prow in the default namespace * reconcile infra changes - one * add missing certificates map & entries * reconcile gke-build and gke-build-trusted clusters * fix eks prow build cluster IAM changes * add tide permissions --------- Co-authored-by: upodroid <upodroid@users.noreply.github.com>
kubernetes · Aug 29, 2024 · bcbde19 · bcbde19
1 parent 5cebca4
commit bcbde19
Show file tree

Hide file tree

Showing 16 changed files with 174 additions and 47 deletions.
diff --git a/infra/aws/terraform/prow-build-cluster/iam.tf b/infra/aws/terraform/prow-build-cluster/iam.tf
@@ -28,20 +28,14 @@ resource "aws_iam_role" "eks_prow_admin" {
       {
         "Effect" : "Allow",
         "Principal" : {
-          "Federated" : aws_iam_openid_connect_provider.k8s_prow[0].arn
+          "Federated" : aws_iam_openid_connect_provider.gke_utility_cluster.arn
         },
         "Action" : "sts:AssumeRoleWithWebIdentity",
         "Condition" : {
           "StringEquals" : {
-            "container.googleapis.com/v1/projects/k8s-prow/locations/us-central1-f/clusters/prow:sub" : [
-              // https://github.com/kubernetes/test-infra/tree/master/config/prow/cluster
-              // all services that load kubeconfig should be listed here
-              "system:serviceaccount:default:deck",
-              "system:serviceaccount:default:config-bootstrapper",
-              "system:serviceaccount:default:crier",
-              "system:serviceaccount:default:sinker",
-              "system:serviceaccount:default:prow-controller-manager",
-              "system:serviceaccount:default:hook"
+            "container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility:sub" : [
+              "system:serviceaccount:argocd:argocd-application-controller",
+              "system:serviceaccount:argocd:argocd-server",
             ]
           }
         }
@@ -55,6 +49,8 @@ resource "aws_iam_role" "eks_prow_admin" {
         "Condition" : {
           "StringEquals" : {
             "container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/prow:sub" : [
+              // https://github.com/kubernetes/k8s.io/tree/main/kubernetes/gke-prow/prow
+              // all services that load kubeconfig should be listed here
               "system:serviceaccount:default:deck",
               "system:serviceaccount:default:config-bootstrapper",
               "system:serviceaccount:default:crier",

diff --git a/infra/aws/terraform/prow-build-cluster/providers.tf b/infra/aws/terraform/prow-build-cluster/providers.tf
@@ -17,7 +17,7 @@ limitations under the License.
 terraform {
   backend "s3" {}
 
-  required_version = "~> 1.5.0"
+  required_version = "~> 1.6"
 
   required_providers {
     aws = {
@@ -47,25 +47,18 @@ provider "aws" {
 provider "kubernetes" {
   host                   = module.eks.cluster_endpoint
   cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+  token                  = data.aws_eks_cluster_auth.eks.token
+}
 
-  # This requires the awscli to be installed locally where Terraform is executed.
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    command     = "aws"
-    args        = local.aws_cli_args
-  }
+
+data "aws_eks_cluster_auth" "eks" {
+  name = module.eks.cluster_name
 }
 
 provider "helm" {
   kubernetes {
     host                   = module.eks.cluster_endpoint
     cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
-
-    # This requires the awscli to be installed locally where Terraform is executed.
-    exec {
-      api_version = "client.authentication.k8s.io/v1beta1"
-      command     = "aws"
-      args        = local.aws_cli_args
-    }
+    token                  = data.aws_eks_cluster_auth.eks.token
   }
 }
diff --git a/infra/aws/terraform/prow-build-cluster/prow.tf b/infra/aws/terraform/prow-build-cluster/prow.tf
@@ -18,18 +18,17 @@ limitations under the License.
 # Provisioning those resources for canary installation is skipped.
 
 # Recognize federated identities from the prow trusted cluster
-resource "aws_iam_openid_connect_provider" "k8s_prow" {
+
+resource "aws_iam_openid_connect_provider" "k8s_infra_prow" {
   count = local.configure_prow ? 1 : 0
 
-  url             = "https://container.googleapis.com/v1/projects/k8s-prow/locations/us-central1-f/clusters/prow"
+  url             = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/prow"
   client_id_list  = ["sts.amazonaws.com"]
   thumbprint_list = ["08745487e891c19e3078c1f2a07e452950ef36f6"]
 }
 
-resource "aws_iam_openid_connect_provider" "k8s_infra_prow" {
-  count = local.configure_prow ? 1 : 0
-
-  url             = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/prow"
+resource "aws_iam_openid_connect_provider" "gke_utility_cluster" {
+  url             = "https://container.googleapis.com/v1/projects/k8s-infra-prow/locations/us-central1/clusters/utility"
   client_id_list  = ["sts.amazonaws.com"]
   thumbprint_list = ["08745487e891c19e3078c1f2a07e452950ef36f6"]
 }
diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/iam.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/iam.tf
@@ -0,0 +1,37 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module "iam" {
+  source  = "terraform-google-modules/iam/google//modules/projects_iam"
+  version = "~> 7"
+
+  projects = [module.project.project_id]
+
+  mode = "authoritative"
+
+  bindings = {
+    "roles/container.admin" = [
+      "group:k8s-infra-cluster-admins@kubernetes.io",
+      "serviceAccount:argocd@k8s-infra-prow.iam.gserviceaccount.com",
+      "serviceAccount:prow-control-plane@k8s-infra-prow.iam.gserviceaccount.com",
+      "serviceAccount:prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com"
+    ]
+    "roles/secretmanager.secretAccessor" = [
+      "serviceAccount:kubernetes-external-secrets@k8s-infra-prow-build-trusted.iam.gserviceaccount.com",
+      "principal://iam.googleapis.com/projects/${module.project.project_number}/locations/global/workloadIdentityPools/${module.project.project_id}.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
+    ]
+  }
+}
diff --git a/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf b/infra/gcp/terraform/k8s-infra-prow-build-trusted/versions.tf
@@ -20,5 +20,5 @@ This file defines:
 */
 
 terraform {
-  required_version = "~> 1.3.0"
+  required_version = "~> 1.6"
 }
diff --git a/infra/gcp/terraform/k8s-infra-prow-build/iam.tf b/infra/gcp/terraform/k8s-infra-prow-build/iam.tf
@@ -14,6 +14,29 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+module "iam" {
+  source  = "terraform-google-modules/iam/google//modules/projects_iam"
+  version = "~> 7"
+
+  projects = [module.project.project_id]
+
+  mode = "authoritative"
+
+  bindings = {
+    "roles/container.admin" = [
+      "group:k8s-infra-cluster-admins@kubernetes.io",
+      "serviceAccount:argocd@k8s-infra-prow.iam.gserviceaccount.com",
+      "serviceAccount:prow-control-plane@k8s-infra-prow.iam.gserviceaccount.com",
+      "serviceAccount:prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com"
+    ]
+    "roles/secretmanager.secretAccessor" = [
+      "serviceAccount:kubernetes-external-secrets@k8s-infra-prow-build.iam.gserviceaccount.com",
+      "principal://iam.googleapis.com/projects/${module.project.project_number}/locations/global/workloadIdentityPools/${module.project.project_id}.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
+    ]
+  }
+}
+
+
 resource "google_iam_workload_identity_pool" "eks_cluster" {
   project = module.project.project_id
 

diff --git a/infra/gcp/terraform/k8s-infra-prow-build/versions.tf b/infra/gcp/terraform/k8s-infra-prow-build/versions.tf
@@ -20,5 +20,5 @@ This file defines:
 */
 
 terraform {
-  required_version = "~> 1.3.0"
+  required_version = "~> 1.6"
 }
diff --git a/infra/gcp/terraform/k8s-infra-prow/buckets.tf b/infra/gcp/terraform/k8s-infra-prow/buckets.tf
@@ -81,3 +81,41 @@ module "testgrid_config_bucket" {
     }
   ]
 }
+
+// Create gs://k8s-testgrid-config to store K8s TestGrid config.
+module "prow_bucket" {
+  source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
+  version = "~> 5"
+
+  name       = "kubernetes-ci-logs"
+  project_id = module.project.project_id
+  location   = "us-central1"
+
+  # TODO: BenTheElder, what lifecycle policy do we have on the previous bucket
+  # lifecycle_rules = [{
+  #   action = {
+  #     type = "Delete"
+  #   }
+  #   condition = {
+  #     age        = 90 # 90d
+  #     with_state = "ANY"
+  #   }
+  # }]
+
+  iam_members = [
+    {
+      // prow pod-utils service account
+      role   = "roles/storage.objectAdmin"
+      member = "serviceAccount:prow-build@k8s-infra-prow-build.iam.gserviceaccount.com"
+    },
+    {
+      role   = "roles/storage.objectAdmin"
+      member = "serviceAccount:${google_service_account.prow.email}"
+    },
+    {
+      role   = "roles/storage.objectViewer"
+      member = "allUsers"
+    },
+  ]
+}
+
diff --git a/infra/gcp/terraform/k8s-infra-prow/certificates.tf b/infra/gcp/terraform/k8s-infra-prow/certificates.tf
@@ -18,18 +18,29 @@ resource "google_certificate_manager_dns_authorization" "prow" {
   name        = "dns-authz-prow-k8s-io"
   description = "*.prow.k8s.io challenge"
   domain      = "prow.k8s.io"
-  project = module.project.project_id
+  project     = module.project.project_id
 }
 
 resource "google_certificate_manager_certificate" "prow" {
   name        = "prow-certificates"
   description = "Prow Certificates"
-  project = module.project.project_id
+  project     = module.project.project_id
   managed {
-    domains = ["prow.k8s.io","*.prow.k8s.io"]
+    domains = ["prow.k8s.io", "*.prow.k8s.io"]
     dns_authorizations = [
       google_certificate_manager_dns_authorization.prow.id
     ]
   }
 }
 
+resource "google_certificate_manager_certificate_map" "prow" {
+  project = module.project.project_id
+  name    = "prow-certificates"
+}
+resource "google_certificate_manager_certificate_map_entry" "prow" {
+  project      = module.project.project_id
+  name         = "prow-certificates"
+  map          = google_certificate_manager_certificate_map.prow.name
+  certificates = [google_certificate_manager_certificate.prow.id]
+  matcher      = "PRIMARY"
+}
diff --git a/infra/gcp/terraform/k8s-infra-prow/gke.tf b/infra/gcp/terraform/k8s-infra-prow/gke.tf
@@ -48,12 +48,12 @@ module "prow" {
   node_pools = [
     {
       name               = "prod-v1"
-      machine_type       = "c3-standard-4"
-      node_locations     = "us-central1-a,us-central1-b"
-      min_count          = 1
-      max_count          = 6
+      machine_type       = "c4-standard-16"
+      node_locations     = "us-central1-a,us-central1-b,us-central1-c"
+      min_count          = 2
+      max_count          = 3
       disk_size_gb       = 100
-      disk_type          = "pd-ssd"
+      disk_type          = "hyperdisk-balanced"
       image_type         = "COS_CONTAINERD"
       auto_repair        = true
       auto_upgrade       = true

diff --git a/infra/gcp/terraform/k8s-infra-prow/iam.tf b/infra/gcp/terraform/k8s-infra-prow/iam.tf
@@ -28,6 +28,7 @@ module "iam" {
     ]
     "roles/container.admin" = [
       "serviceAccount:${google_service_account.argocd.email}",
+      "serviceAccount:${google_service_account.prow.email}",
       "principal://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/k8s-infra-prow.svc.id.goog/subject/ns/argocd/sa/argocd-application-controller",
       "principal://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/k8s-infra-prow.svc.id.goog/subject/ns/argocd/sa/argocd-applicationset-controller",
       "principal://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/k8s-infra-prow.svc.id.goog/subject/ns/argocd/sa/argocd-server",
@@ -69,6 +70,17 @@ resource "google_service_account" "argocd" {
   project      = module.project.project_id
 }
 
+resource "google_service_account_iam_binding" "argocd" {
+  service_account_id = google_service_account.argocd.name
+  role               = "roles/iam.workloadIdentityUser"
+
+  members = [
+    "serviceAccount:k8s-infra-prow.svc.id.goog[argocd/argocd-application-controller]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[argocd/argocd-server]",
+  ]
+}
+
+
 resource "google_service_account" "image_builder" {
   account_id   = "image-builder"
   display_name = "Image Builder"
@@ -80,3 +92,23 @@ resource "google_service_account_iam_member" "image_builder" {
   role               = "roles/iam.serviceAccountUser"
   member             = "serviceAccount:gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com"
 }
+
+resource "google_service_account" "prow" {
+  account_id   = "prow-control-plane"
+  display_name = "Prow Control Plane"
+  project      = module.project.project_id
+}
+
+resource "google_service_account_iam_binding" "prow" {
+  service_account_id = google_service_account.prow.name
+  role               = "roles/iam.workloadIdentityUser"
+  members = [
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/config-bootstrapper]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/crier]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/deck]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/hook]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/prow-controller-manager]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/sinker]",
+    "serviceAccount:k8s-infra-prow.svc.id.goog[default/tide]",
+  ]
+}
diff --git a/infra/gcp/terraform/modules/gke-cluster/versions.tf b/infra/gcp/terraform/modules/gke-cluster/versions.tf
@@ -17,8 +17,6 @@
  */
 
 terraform {
-  required_version = "~> 1.3.0"
-
   required_providers {
     google = {
       source  = "hashicorp/google"

diff --git a/infra/gcp/terraform/modules/gke-nodepool/versions.tf b/infra/gcp/terraform/modules/gke-nodepool/versions.tf
@@ -17,8 +17,6 @@
  */
 
 terraform {
-  required_version = "~> 1.3.0"
-
   required_providers {
     google = {
       source  = "hashicorp/google"

diff --git a/infra/gcp/terraform/modules/gke-project/outputs.tf b/infra/gcp/terraform/modules/gke-project/outputs.tf
@@ -18,3 +18,8 @@ output "project_id" {
   description = "The project_id of the project that was created"
   value       = google_project.project.project_id
 }
+
+output "project_number" {
+  description = "Numeric identifier for the project"
+  value       = google_project.project.number
+}
diff --git a/infra/gcp/terraform/modules/gke-project/versions.tf b/infra/gcp/terraform/modules/gke-project/versions.tf
@@ -17,8 +17,6 @@
  */
 
 terraform {
-  required_version = "~> 1.3.0"
-
   required_providers {
     google = {
       source  = "hashicorp/google"

diff --git a/infra/gcp/terraform/modules/workload-identity-service-account/versions.tf b/infra/gcp/terraform/modules/workload-identity-service-account/versions.tf
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 terraform {
-  required_version = "~> 1.3.0"
   required_providers {
     google = {
       source  = "hashicorp/google"