diff --git a/modules/acm/README.md b/modules/acm/README.md index 65ddaa575..71f2d61fb 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -68,5 +68,6 @@ By default, this module will attempt to download the ACM operator from Google di | Name | Description | |------|-------------| | git\_creds\_public | Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository. | +| wait | An output to use when you want to depend on cmd finishing | diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf index 5a1bdd47e..6bb2a808e 100644 --- a/modules/acm/outputs.tf +++ b/modules/acm/outputs.tf @@ -18,3 +18,8 @@ output "git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository." value = module.acm_operator.git_creds_public } + +output "wait" { + description = "An output to use when you want to depend on cmd finishing" + value = module.acm_operator.wait +} diff --git a/modules/hub/README.md b/modules/hub/README.md index 2b4ad0bff..3bd8620c6 100644 --- a/modules/hub/README.md +++ b/modules/hub/README.md @@ -40,6 +40,7 @@ To deploy this config: | gke\_hub\_membership\_name | Memebership name that uniquely represents the cluster being registered on the Hub | string | `"gke-hub-membership"` | no | | gke\_hub\_sa\_name | Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace. | string | `"gke-hub-sa"` | no | | location | The location (zone or region) this cluster has been created in. | string | n/a | yes | +| module\_depends\_on | List of modules or resources this module depends on. | list | `` | no | | project\_id | The project in which the resource belongs. | string | n/a | yes | | sa\_private\_key | Private key for service account base64 encoded. Required only if `use_existing_sa` is set to `true`. | string | `"null"` | no | | skip\_gcloud\_download | Whether to skip downloading gcloud (assumes gcloud and kubectl already available outside the module) | bool | `"true"` | no | diff --git a/modules/hub/main.tf b/modules/hub/main.tf index d8ac255f1..30b73774d 100644 --- a/modules/hub/main.tf +++ b/modules/hub/main.tf @@ -49,7 +49,7 @@ module "gke_hub_registration" { skip_download = var.skip_gcloud_download upgrade = true use_tf_google_credentials_env_var = var.use_tf_google_credentials_env_var - module_depends_on = [var.cluster_endpoint] + module_depends_on = concat([var.cluster_endpoint], var.module_depends_on) create_cmd_entrypoint = "${path.module}/scripts/gke_hub_registration.sh" create_cmd_body = "${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key} ${var.project_id}" diff --git a/modules/hub/variables.tf b/modules/hub/variables.tf index aea985505..4fdf08a28 100644 --- a/modules/hub/variables.tf +++ b/modules/hub/variables.tf @@ -81,3 +81,9 @@ variable "sa_private_key" { type = string default = null } + +variable "module_depends_on" { + description = "List of modules or resources this module depends on." + type = list + default = [] +} diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf index 6d14e092d..7178a4829 100644 --- a/modules/k8s-operator-crd-support/main.tf +++ b/modules/k8s-operator-crd-support/main.tf @@ -113,3 +113,19 @@ module "k8sop_config" { kubectl_create_command = "kubectl apply -f ${local_file.operator_cr.filename}" kubectl_destroy_command = "kubectl delete -f ${local_file.operator_cr.filename}" } + +module "wait_for_gatekeeper" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 2.0.2" + enabled = var.enable_policy_controller ? true : false + module_depends_on = [module.k8sop_config.wait] + skip_download = var.skip_gcloud_download + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + create_cmd_triggers = { script_sha1 = sha1(file("${path.module}/scripts/wait_for_gatekeeper.sh")) } + service_account_key_file = var.service_account_key_file + + kubectl_create_command = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location}" + kubectl_destroy_command = "" +} diff --git a/modules/k8s-operator-crd-support/outputs.tf b/modules/k8s-operator-crd-support/outputs.tf index 66f3aa043..6b4f7e321 100644 --- a/modules/k8s-operator-crd-support/outputs.tf +++ b/modules/k8s-operator-crd-support/outputs.tf @@ -19,5 +19,10 @@ output "git_creds_public" { value = var.create_ssh_key ? tls_private_key.k8sop_creds.*.public_key_openssh : null } +output "wait" { + description = "An output to use when you want to depend on cmd finishing" + value = var.enable_policy_controller ? module.wait_for_gatekeeper.wait : module.k8sop_config.wait +} + diff --git a/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh new file mode 100755 index 000000000..8fde4852e --- /dev/null +++ b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +is_deployment_ready() { + kubectl --context "$1" -n "$2" get deploy "$3" &> /dev/null + export exit_code=$? + while [ ! " ${exit_code} " -eq 0 ] + do + sleep 5 + echo -e "Waiting for deployment $3 in cluster $1 to be created..." + kubectl --context "$1" -n "$2" get deploy "$3" &> /dev/null + export exit_code=$? + done + echo -e "Deployment $3 in cluster $1 created." + + # Once deployment is created, check for deployment status.availableReplicas is greater than 0 + availableReplicas=$(kubectl --context "$1" -n "$2" get deploy "$3" -o json | jq -r '.status.availableReplicas') + while [[ " ${availableReplicas} " == " null " ]] + do + sleep 5 + echo -e "Waiting for deployment $3 in cluster $1 to become ready..." + availableReplicas=$(kubectl --context "$1" -n "$2" get deploy "$3" -o json | jq -r '.status.availableReplicas') + done + + echo -e "$3 in cluster $1 is ready with replicas ${availableReplicas}." + return "${availableReplicas}" +} + +is_service_ready() { + kubectl --context "$1" -n "$2" get service "$3" &> /dev/null + export exit_code=$? + while [ ! " ${exit_code} " -eq 0 ] + do + sleep 5 + echo -e "Waiting for service $3 in cluster $1 to be created..." + kubectl --context "$1" -n "$2" get service "$3" &> /dev/null + export exit_code=$? + done + echo -e "Service $3 in cluster $1 created." + + # Once service is created, check endpoints is greater than 0 + kubectl --context "$1" -n "$2" get endpoints "$3" + export exit_code=$? + + while [ ! " ${exit_code} " -eq 0 ] + do + sleep 5 + echo -e "Waiting for endpoints for service $3 in cluster $1 to become ready..." + kubectl --context "$1" -n "$2" get endpoints "$3" + export exit_code=$? + done + + echo -e "Service $3 in cluster $1 is ready with endpoints." + return +} + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +PROJECT_ID=$1 +CLUSTER_NAME=$2 +CLUSTER_LOCATION=$3 + +# Gatekeeper causes issues if not ready +is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager +is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service