Merge pull request #885 from spiffxp/add-greenhouse

Add greenhouse to k8s-infra-prow-build

infra/gcp/clusters/modules/gke-nodepool/main.tf

@@ -44,6 +44,8 @@ resource "google_container_node_pool" "node_pool" {
     machine_type = var.machine_type
     disk_size_gb = var.disk_size_gb
     disk_type    = var.disk_type
+    labels       = var.labels
+    taint        = var.taints
 
     service_account = var.service_account
     oauth_scopes    = ["https://www.googleapis.com/auth/cloud-platform"]
@@ -61,5 +63,9 @@ resource "google_container_node_pool" "node_pool" {
   // the old one
   lifecycle {
     create_before_destroy = true
+    # https://www.terraform.io/docs/providers/google/r/container_cluster.html#taint
+    ignore_changes = [
+      node_config["taint"],
+    ]
   }
 }

infra/gcp/clusters/modules/gke-nodepool/variables.tf

@@ -64,6 +64,27 @@ variable "disk_type" {
   type        = string
 }
 
+variable "labels" {
+  description = "The labels to apply to this node_pool"
+  type        = map(string)
+  default     = {}
+}
+
+# Terraform docs suggest not using terraform to manage taints, because GKE is going
+# to auto-apply taints if certain features are enabled, and terraform doesn't do well
+# when something else is managing the same thing it's managing.
+#
+# So this is mostly here to describe intent. It will assign taints at creation time,
+# but cannot be used to ensure the taints remain applied throughout the node pool's
+# lifecycle
+#
+# ref: https://www.terraform.io/docs/providers/google/r/container_cluster.html#taint
+variable "taints" {
+  description = "The taints to apply to this node_pool upon creation (NOTE: changes will be ignored throughout lifecycle)"
+  type        = list(object({ key = string, value = string, effect = string }))
+  default     = []
+}
+
 variable "service_account" {
   description = "The email address of the GCP Service Account to be associated with nodes in this node_pool"
   type        = string

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf

@@ -110,3 +110,23 @@ module "prow_build_nodepool" {
   disk_type       = "pd-ssd"
   service_account = module.prow_build_cluster.cluster_node_sa.email
 }
+
+module "greenhouse_nodepool" {
+  source = "../../../modules/gke-nodepool"
+  project_name    = local.project_id
+  cluster_name    = module.prow_build_cluster.cluster.name
+  location        = module.prow_build_cluster.cluster.location
+  name            = "greenhouse"
+  labels          = { dedicated = "greenhouse" }
+  # NOTE: taints are only applied during creation and ignored after that, see module docs
+  taints          = [{ key = "dedicated", value = "greenhouse", effect = "NO_SCHEDULE" }]
+  min_count       = 1
+  max_count       = 1
+  # choosing this image for parity with the build nodepool
+  image_type      = "UBUNTU_CONTAINERD"
+  # choosing a machine type to maximize IOPs
+  machine_type    = "n1-standard-32"
+  disk_size_gb    = 100
+  disk_type       = "pd-standard"
+  service_account = module.prow_build_cluster.cluster_node_sa.email
+}

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/resources/greenhouse-deployment.yaml

@@ -0,0 +1,58 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: greenhouse
+  namespace: default
+  labels:
+    app: greenhouse
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: greenhouse
+  template:
+    metadata:
+      labels:
+        app: greenhouse
+    spec:
+      containers:
+      - name: greenhouse
+        image: gcr.io/k8s-testimages/greenhouse:latest
+        imagePullPolicy: Always
+        ports:
+        - name: cache
+          containerPort: 8080
+        - name: metrics
+          containerPort: 9090
+        args:
+        - --dir=/data
+        - --min-percent-blocks-free=2
+        volumeMounts:
+        - name: cache
+          mountPath: /data
+      volumes:
+      - name: cache
+        persistentVolumeClaim:
+          claimName: greenhouse
+      # run on our dedicated node
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "greenhouse"
+        effect: "NoSchedule"
+      nodeSelector:
+        dedicated: "greenhouse"

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/resources/greenhouse-metrics-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: bazel-cache-metrics
+  namespace: default
+spec:
+  selector:
+    app: greenhouse
+  ports:
+  - name: default
+    protocol: TCP
+    port: 80
+    targetPort: 9090
+  loadBalancerIP: 34.72.140.202
+  type: LoadBalancer

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/resources/greenhouse-service.yaml

@@ -0,0 +1,27 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: bazel-cache
+  namespace: default
+  labels:
+    run: bazel-cache
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+  selector:
+    app: greenhouse

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/resources/greenhouse-storage.yaml

@@ -0,0 +1,31 @@
+# storage class used by greenhouse for GKE / GCE we use persistent SSD
+# previously we also used local SSDs via hostPath which are *great* but
+# "only" ~375 GB
+# https://cloud.google.com/compute/docs/disks/
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: greenhouse
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd
+# we want to use a volume with strictatime,lazytime (and not noatime or relatime)
+# so that file access times *are* recorded but are lazily flushed to the disk
+# https://lwn.net/Articles/621046/
+# https://unix.stackexchange.com/questions/276858/why-is-ext4-filesystem-mounted-with-both-relatime-and-lazytime
+mountOptions: ["strictatime", "lazytime"]
+---
+# 3TB of SSD :-)
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: greenhouse
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 3000Gi
+  storageClassName: greenhouse
+---

infra/gcp/ensure-e2e-projects.sh

@@ -48,6 +48,15 @@ ensure_regional_address \
   "to allow monitoring.k8s.prow.io to scrape boskos metrics"
 ) 2>&1 | indent
 
+color 6 "Ensuring greenhouse is empowered"
+(
+ensure_regional_address \
+  "k8s-infra-prow-build" \
+  "us-central1" \
+  "greenhouse-metrics" \
+  "to allow monitoring.k8s.prow.io to scrape greenhouse metrics"
+) 2>&1 | indent
+
 ## setup projects to be used by e2e tests for standing up clusters
 
 E2E_MANUAL_PROJECTS=(