From 7614a709b13e542c2bb16f162b79d0c4b46cf2ac Mon Sep 17 00:00:00 2001 From: Abhinav Rau Date: Mon, 23 Nov 2020 06:45:13 +0000 Subject: [PATCH 1/5] Tested with EKS and GKE --- modules/acm/main.tf | 36 +++++++++---------- modules/acm/variables.tf | 6 ++++ modules/k8s-operator-crd-support/main.tf | 26 ++++++++------ .../scripts/wait_for_gatekeeper.sh | 14 ++++++-- modules/k8s-operator-crd-support/variables.tf | 6 ++++ 5 files changed, 58 insertions(+), 30 deletions(-) diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 244816f99e..f81ccb102e 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -18,24 +18,24 @@ module "acm_operator" { source = "../k8s-operator-crd-support" - cluster_name = var.cluster_name - project_id = var.project_id - location = var.location - operator_path = var.operator_path - sync_repo = var.sync_repo - sync_branch = var.sync_branch - policy_dir = var.policy_dir - cluster_endpoint = var.cluster_endpoint - create_ssh_key = var.create_ssh_key - secret_type = var.secret_type - ssh_auth_key = var.ssh_auth_key - enable_policy_controller = var.enable_policy_controller - install_template_library = var.install_template_library - source_format = var.source_format - hierarchy_controller = var.hierarchy_controller - enable_log_denies = var.enable_log_denies - service_account_key_file = var.service_account_key_file - + cluster_name = var.cluster_name + project_id = var.project_id + location = var.location + operator_path = var.operator_path + sync_repo = var.sync_repo + sync_branch = var.sync_branch + policy_dir = var.policy_dir + cluster_endpoint = var.cluster_endpoint + create_ssh_key = var.create_ssh_key + secret_type = var.secret_type + ssh_auth_key = var.ssh_auth_key + enable_policy_controller = var.enable_policy_controller + install_template_library = var.install_template_library + source_format = var.source_format + hierarchy_controller = var.hierarchy_controller + enable_log_denies = var.enable_log_denies + service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context operator_latest_manifest_url = "gs://config-management-release/released/latest/config-management-operator.yaml" operator_cr_template_path = "${path.module}/templates/acm-config.yml.tpl" operator_credential_namespace = "config-management-system" diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index acc6644ec1..ac8f134789 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -109,3 +109,9 @@ variable "service_account_key_file" { description = "Path to service account key file to auth as for running `gcloud container clusters get-credentials`." default = "" } + +variable "use_existing_context" { + description = "Use existing kubecontext to auth kube-api." + type = bool + default = false +} diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf index 334b0d23bf..a6549ce1bf 100644 --- a/modules/k8s-operator-crd-support/main.tf +++ b/modules/k8s-operator-crd-support/main.tf @@ -15,15 +15,17 @@ */ locals { - cluster_endpoint = "https://${var.cluster_endpoint}" - private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key - k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type - should_download_manifest = var.operator_path == null ? true : false - manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path - sync_branch_node = var.sync_branch != "" ? format("syncBranch: %s", var.sync_branch) : "" - policy_dir_node = var.policy_dir != "" ? format("policyDir: %s", var.policy_dir) : "" - hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchy_controller:\n %s", yamlencode(var.hierarchy_controller)) - source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : "" + cluster_endpoint = "https://${var.cluster_endpoint}" + private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key + k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type + should_download_manifest = var.operator_path == null ? true : false + manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path + sync_branch_node = var.sync_branch != "" ? format("syncBranch: %s", var.sync_branch) : "" + policy_dir_node = var.policy_dir != "" ? format("policyDir: %s", var.policy_dir) : "" + hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchy_controller:\n %s", yamlencode(var.hierarchy_controller)) + source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : "" + append_arg_use_existing_context_for_gatekeeper = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : "" + } module "k8sop_manifest" { @@ -46,6 +48,7 @@ module "k8s_operator" { cluster_location = var.location project_id = var.project_id service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context kubectl_create_command = "kubectl apply -f ${local.manifest_path}" kubectl_destroy_command = "kubectl delete -f ${local.manifest_path}" @@ -68,6 +71,7 @@ module "k8sop_creds_secret" { cluster_location = var.location project_id = var.project_id service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context kubectl_create_command = local.private_key != null ? "kubectl create secret generic ${var.operator_credential_name} -n=${var.operator_credential_namespace} --from-literal=${local.k8sop_creds_secret_key}='${local.private_key}'" : "" kubectl_destroy_command = "kubectl delete secret ${var.operator_credential_name} -n=${var.operator_credential_namespace}" @@ -105,6 +109,7 @@ module "k8sop_config" { project_id = var.project_id create_cmd_triggers = { configmanagement = local_file.operator_cr.content } service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context kubectl_create_command = "kubectl apply -f ${local_file.operator_cr.filename}" kubectl_destroy_command = "kubectl delete -f ${local_file.operator_cr.filename}" @@ -120,7 +125,8 @@ module "wait_for_gatekeeper" { project_id = var.project_id create_cmd_triggers = { script_sha1 = sha1(file("${path.module}/scripts/wait_for_gatekeeper.sh")) } service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context - kubectl_create_command = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location}" + kubectl_create_command = format("%s %s", "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location}", local.append_arg_use_existing_context_for_gatekeeper) kubectl_destroy_command = "" } diff --git a/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh index 8fde4852ea..3dcdf1b4b2 100755 --- a/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh +++ b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh @@ -74,7 +74,17 @@ fi PROJECT_ID=$1 CLUSTER_NAME=$2 CLUSTER_LOCATION=$3 +USE_EXISTING_CONTEXT=$4 # Gatekeeper causes issues if not ready -is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager -is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service + +# Check if we need to use the current context +if [[ -n {USE_EXISTING_CONTEXT} ]]; then + # Get the current context + CURRENT_CONTEXT=$(kubectl config current-context) + is_deployment_ready "${CURRENT_CONTEXT}" gatekeeper-system gatekeeper-controller-manager + is_service_ready "${CURRENT_CONTEXT}" gatekeeper-system gatekeeper-webhook-service +else + is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager + is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service +fi \ No newline at end of file diff --git a/modules/k8s-operator-crd-support/variables.tf b/modules/k8s-operator-crd-support/variables.tf index 431ae9ee7d..078cdc8081 100644 --- a/modules/k8s-operator-crd-support/variables.tf +++ b/modules/k8s-operator-crd-support/variables.tf @@ -133,3 +133,9 @@ variable "service_account_key_file" { description = "Path to service account key file to auth as for running `gcloud container clusters get-credentials`." default = "" } + +variable "use_existing_context" { + description = "Use existing kubecontext to auth kube-api. Useful when working with to non GKE clusters" + type = bool + default = false +} From 2ba43e164805bd06fbd0ca2708fee9437b29cbd6 Mon Sep 17 00:00:00 2001 From: Abhinav Rau Date: Mon, 23 Nov 2020 17:16:33 +0000 Subject: [PATCH 2/5] Reformatting to support PR for ACM support for non GKE Clusters --- modules/acm/main.tf | 37 ++++++++++++------------ modules/k8s-operator-crd-support/main.tf | 1 - 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/acm/main.tf b/modules/acm/main.tf index f81ccb102e..c7e88f0c3d 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -18,24 +18,25 @@ module "acm_operator" { source = "../k8s-operator-crd-support" - cluster_name = var.cluster_name - project_id = var.project_id - location = var.location - operator_path = var.operator_path - sync_repo = var.sync_repo - sync_branch = var.sync_branch - policy_dir = var.policy_dir - cluster_endpoint = var.cluster_endpoint - create_ssh_key = var.create_ssh_key - secret_type = var.secret_type - ssh_auth_key = var.ssh_auth_key - enable_policy_controller = var.enable_policy_controller - install_template_library = var.install_template_library - source_format = var.source_format - hierarchy_controller = var.hierarchy_controller - enable_log_denies = var.enable_log_denies - service_account_key_file = var.service_account_key_file - use_existing_context = var.use_existing_context + cluster_name = var.cluster_name + project_id = var.project_id + location = var.location + operator_path = var.operator_path + sync_repo = var.sync_repo + sync_branch = var.sync_branch + policy_dir = var.policy_dir + cluster_endpoint = var.cluster_endpoint + create_ssh_key = var.create_ssh_key + secret_type = var.secret_type + ssh_auth_key = var.ssh_auth_key + enable_policy_controller = var.enable_policy_controller + install_template_library = var.install_template_library + source_format = var.source_format + hierarchy_controller = var.hierarchy_controller + enable_log_denies = var.enable_log_denies + service_account_key_file = var.service_account_key_file + use_existing_context = var.use_existing_context + operator_latest_manifest_url = "gs://config-management-release/released/latest/config-management-operator.yaml" operator_cr_template_path = "${path.module}/templates/acm-config.yml.tpl" operator_credential_namespace = "config-management-system" diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf index a6549ce1bf..09f056cd02 100644 --- a/modules/k8s-operator-crd-support/main.tf +++ b/modules/k8s-operator-crd-support/main.tf @@ -25,7 +25,6 @@ locals { hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchy_controller:\n %s", yamlencode(var.hierarchy_controller)) source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : "" append_arg_use_existing_context_for_gatekeeper = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : "" - } module "k8sop_manifest" { From f77a6088ac8acb426f7881f3c8e65e44bdc78f55 Mon Sep 17 00:00:00 2001 From: Abhinav Rau Date: Tue, 12 Jan 2021 21:14:54 +0000 Subject: [PATCH 3/5] update docs --- modules/acm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/acm/README.md b/modules/acm/README.md index f4dab2a0f2..b152152396 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -62,6 +62,7 @@ By default, this module will attempt to download the ACM operator from Google di | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no | | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no | | sync\_repo | ACM Git repo address | `string` | n/a | yes | +| use\_existing\_context | Use existing kubecontext to auth kube-api. | `bool` | `false` | no | ## Outputs From db23ea8441b3db23a3f9878615e8c4be1525f7bf Mon Sep 17 00:00:00 2001 From: Abhinav Rau Date: Tue, 12 Jan 2021 22:05:19 +0000 Subject: [PATCH 4/5] fix shellcheck errors --- .../scripts/wait_for_gatekeeper.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh index 5413014359..108d2d4bca 100755 --- a/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh +++ b/modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh @@ -79,12 +79,14 @@ USE_EXISTING_CONTEXT=$4 # Gatekeeper causes issues if not ready # Check if we need to use the current context -if [[ -n {USE_EXISTING_CONTEXT} ]]; then - # Get the current context +if [ -z ${USE_EXISTING_CONTEXT+x} ]; then + # GKE Cluster. Use the GKE cluster context + is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager + is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service +else + echo "USE_EXISTING_CONTEXT variable is set. Using current context to wait for deployment to be ready." + # Get the current context. This can be used for non GKE Clusters CURRENT_CONTEXT=$(kubectl config current-context) is_deployment_ready "${CURRENT_CONTEXT}" gatekeeper-system gatekeeper-controller-manager is_service_ready "${CURRENT_CONTEXT}" gatekeeper-system gatekeeper-webhook-service -else - is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager - is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service -fi \ No newline at end of file +fi From a6e1b173260728e09636bab956359e4d4a882c97 Mon Sep 17 00:00:00 2001 From: Abhinav Rau Date: Wed, 13 Jan 2021 07:09:53 -0500 Subject: [PATCH 5/5] Update modules/k8s-operator-crd-support/main.tf Remove unneeded format call. Co-authored-by: Bharath KKB --- modules/k8s-operator-crd-support/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf index fe76bf123b..50b3f3ec13 100644 --- a/modules/k8s-operator-crd-support/main.tf +++ b/modules/k8s-operator-crd-support/main.tf @@ -121,6 +121,6 @@ module "wait_for_gatekeeper" { service_account_key_file = var.service_account_key_file use_existing_context = var.use_existing_context - kubectl_create_command = format("%s %s", "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location}", local.append_arg_use_existing_context_for_gatekeeper) + kubectl_create_command = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context_for_gatekeeper}" kubectl_destroy_command = "" }