From 23da1035cf55bb4a2e371c905387d040a3748ac5 Mon Sep 17 00:00:00 2001 From: unlimitedbits <70005860+unlimitedbits@users.noreply.github.com> Date: Mon, 17 May 2021 06:39:32 +0200 Subject: [PATCH] feat!: Add multi-repo support for Config Sync (#872) * Add multi-repo support for Config Sync * Run make build * increase wait time for configsync api * Update modules/k8s-operator-crd-support/main.tf Co-authored-by: Bharath KKB * Enhance wait_for_configsync_api step Co-authored-by: Morgante Pell Co-authored-by: Bharath KKB Co-authored-by: Morgante Pell --- modules/acm/README.md | 4 +- modules/acm/main.tf | 4 + modules/acm/templates/acm-config.yml.tpl | 5 ++ modules/acm/templates/root-sync.yml.tpl | 14 ++++ modules/acm/variables.tf | 14 +++- modules/config-sync/README.md | 4 +- modules/config-sync/main.tf | 4 + .../config-sync/templates/root-sync.yml.tpl | 14 ++++ modules/config-sync/variables.tf | 14 +++- modules/k8s-operator-crd-support/main.tf | 81 ++++++++++++++++--- .../scripts/wait_for_configsync.sh | 64 +++++++++++++++ modules/k8s-operator-crd-support/variables.tf | 25 +++++- 12 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 modules/acm/templates/root-sync.yml.tpl create mode 100644 modules/config-sync/templates/root-sync.yml.tpl create mode 100755 modules/k8s-operator-crd-support/scripts/wait_for_configsync.sh diff --git a/modules/acm/README.md b/modules/acm/README.md index b15215239..b996d286a 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -49,8 +49,9 @@ By default, this module will attempt to download the ACM operator from Google di | cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no | | enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no | +| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no | | enable\_policy\_controller | Whether to enable the ACM Policy Controller on the cluster | `bool` | `true` | no | -| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map` | `null` | no | +| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map(any)` | `null` | no | | install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no | | location | GCP location used to reach cluster. | `string` | n/a | yes | | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no | @@ -62,6 +63,7 @@ By default, this module will attempt to download the ACM operator from Google di | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no | | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no | | sync\_repo | ACM Git repo address | `string` | n/a | yes | +| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no | | use\_existing\_context | Use existing kubecontext to auth kube-api. | `bool` | `false` | no | ## Outputs diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 419abf615..7acc98662 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -37,8 +37,10 @@ module "acm_operator" { project_id = var.project_id location = var.location operator_path = var.operator_path + enable_multi_repo = var.enable_multi_repo sync_repo = var.sync_repo sync_branch = var.sync_branch + sync_revision = var.sync_revision policy_dir = var.policy_dir cluster_endpoint = var.cluster_endpoint create_ssh_key = var.create_ssh_key @@ -56,4 +58,6 @@ module "acm_operator" { operator_cr_template_path = "${path.module}/templates/acm-config.yml.tpl" operator_credential_namespace = "config-management-system" operator_credential_name = "git-creds" + + rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl" } diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl index 5059c1df2..e8ebeb966 100644 --- a/modules/acm/templates/acm-config.yml.tpl +++ b/modules/acm/templates/acm-config.yml.tpl @@ -9,10 +9,15 @@ spec: enabled: ${enable_policy_controller} templateLibraryInstalled: ${install_template_library} logDeniesEnabled: ${enable_log_denies} +%{ if enable_multi_repo ~} + enableMultiRepo: true +%{ else ~} git: syncRepo: ${sync_repo} secretType: ${secret_type} ${policy_dir_node} ${sync_branch_node} + ${sync_revision_node} ${source_format_node} +%{ endif ~} ${hierarchy_controller_map_node} diff --git a/modules/acm/templates/root-sync.yml.tpl b/modules/acm/templates/root-sync.yml.tpl new file mode 100644 index 000000000..c2ba6ec52 --- /dev/null +++ b/modules/acm/templates/root-sync.yml.tpl @@ -0,0 +1,14 @@ +apiVersion: configsync.gke.io/v1beta1 +kind: RootSync +metadata: + name: root-sync + namespace: config-management-system +spec: + ${source_format_node} + git: + repo: ${sync_repo} + auth: ${secret_type} + ${policy_dir_node} + ${sync_branch_node} + ${sync_revision_node} + ${secret_ref_node} diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index ac8f13478..dd11baccc 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -35,6 +35,12 @@ variable "operator_path" { default = null } +variable "enable_multi_repo" { + description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)." + type = bool + default = false +} + variable "sync_repo" { description = "ACM Git repo address" type = string @@ -46,6 +52,12 @@ variable "sync_branch" { default = "" } +variable "sync_revision" { + description = "ACM repo Git revision. If un-set, uses Config Management default." + type = string + default = "" +} + variable "policy_dir" { description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default." type = string @@ -95,7 +107,7 @@ variable "source_format" { variable "hierarchy_controller" { description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details" - type = map + type = map(any) default = null } diff --git a/modules/config-sync/README.md b/modules/config-sync/README.md index 8afcd943a..2a2c91bf1 100644 --- a/modules/config-sync/README.md +++ b/modules/config-sync/README.md @@ -50,7 +50,8 @@ To deploy this config: | cluster\_endpoint | Kubernetes cluster endpoint. | `string` | n/a | yes | | cluster\_name | GCP cluster name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no | -| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map` | `null` | no | +| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no | +| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map(any)` | `null` | no | | location | GCP location used to reach cluster. | `string` | n/a | yes | | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no | | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no | @@ -60,6 +61,7 @@ To deploy this config: | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no | | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no | | sync\_repo | ACM Git repo address | `string` | n/a | yes | +| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no | ## Outputs diff --git a/modules/config-sync/main.tf b/modules/config-sync/main.tf index c4f2c23f0..052cacb3b 100644 --- a/modules/config-sync/main.tf +++ b/modules/config-sync/main.tf @@ -22,8 +22,10 @@ module "configsync_operator" { project_id = var.project_id location = var.location operator_path = var.operator_path + enable_multi_repo = var.enable_multi_repo sync_repo = var.sync_repo sync_branch = var.sync_branch + sync_revision = var.sync_revision policy_dir = var.policy_dir cluster_endpoint = var.cluster_endpoint create_ssh_key = var.create_ssh_key @@ -36,4 +38,6 @@ module "configsync_operator" { operator_cr_template_path = "${path.module}/templates/config-sync-config.yml.tpl" operator_credential_namespace = "config-management-system" operator_credential_name = "git-creds" + + rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl" } diff --git a/modules/config-sync/templates/root-sync.yml.tpl b/modules/config-sync/templates/root-sync.yml.tpl new file mode 100644 index 000000000..c2ba6ec52 --- /dev/null +++ b/modules/config-sync/templates/root-sync.yml.tpl @@ -0,0 +1,14 @@ +apiVersion: configsync.gke.io/v1beta1 +kind: RootSync +metadata: + name: root-sync + namespace: config-management-system +spec: + ${source_format_node} + git: + repo: ${sync_repo} + auth: ${secret_type} + ${policy_dir_node} + ${sync_branch_node} + ${sync_revision_node} + ${secret_ref_node} diff --git a/modules/config-sync/variables.tf b/modules/config-sync/variables.tf index b740d6480..512786dc7 100644 --- a/modules/config-sync/variables.tf +++ b/modules/config-sync/variables.tf @@ -35,6 +35,12 @@ variable "operator_path" { default = null } +variable "enable_multi_repo" { + description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)." + type = bool + default = false +} + variable "sync_repo" { description = "ACM Git repo address" type = string @@ -46,6 +52,12 @@ variable "sync_branch" { default = "" } +variable "sync_revision" { + description = "ACM repo Git revision. If un-set, uses Config Management default." + type = string + default = "" +} + variable "policy_dir" { description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default." type = string @@ -82,6 +94,6 @@ variable "source_format" { variable "hierarchy_controller" { description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details." - type = map + type = map(any) default = null } diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf index d91f2f7c1..3d72a2089 100644 --- a/modules/k8s-operator-crd-support/main.tf +++ b/modules/k8s-operator-crd-support/main.tf @@ -15,16 +15,21 @@ */ locals { - cluster_endpoint = "https://${var.cluster_endpoint}" - private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key - k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type - should_download_manifest = var.operator_path == null ? true : false - manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path - sync_branch_node = var.sync_branch != "" ? format("syncBranch: %s", var.sync_branch) : "" - policy_dir_node = var.policy_dir != "" ? format("policyDir: %s", var.policy_dir) : "" - hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchyController:\n %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:"))) - source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : "" - append_arg_use_existing_context_for_gatekeeper = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : "" + cluster_endpoint = "https://${var.cluster_endpoint}" + private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key + k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type + should_download_manifest = var.operator_path == null ? true : false + manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path + sync_branch_property = var.enable_multi_repo ? "branch" : "syncBranch" + sync_branch_node = var.sync_branch != "" ? format("%s: %s", local.sync_branch_property, var.sync_branch) : "" + sync_revision_property = var.enable_multi_repo ? "revision" : "syncRev" + sync_revision_node = var.sync_revision != "" ? format("%s: %s", local.sync_revision_property, var.sync_revision) : "" + policy_dir_property = var.enable_multi_repo ? "dir" : "policyDir" + policy_dir_node = var.policy_dir != "" ? format("%s: %s", local.policy_dir_property, var.policy_dir) : "" + secret_ref_node = var.create_ssh_key == true || var.ssh_auth_key != null ? format("secretRef:\n name: %s", var.operator_credential_name) : "" + hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchyController:\n %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:"))) + source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : "" + append_arg_use_existing_context = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : "" } module "k8sop_manifest" { @@ -82,8 +87,10 @@ data "template_file" "k8sop_config" { template = file(var.operator_cr_template_path) vars = { cluster_name = var.cluster_name + enable_multi_repo = var.enable_multi_repo sync_repo = var.sync_repo sync_branch_node = local.sync_branch_node + sync_revision_node = local.sync_revision_node policy_dir_node = local.policy_dir_node secret_type = var.create_ssh_key ? "ssh" : var.secret_type enable_policy_controller = var.enable_policy_controller ? "true" : "false" @@ -109,6 +116,58 @@ module "k8sop_config" { kubectl_destroy_command = "kubectl delete -f - < /dev/null + export exit_code=$? + while [ ! " ${exit_code} " -eq 0 ] + do + sleep 5 + echo -e "Waiting for namespace config-mangement-system in cluster $1 to be created..." + kubectl --context "$1" get namespace config-management-system &> /dev/null + export exit_code=$? + done + echo -e "Namespace config-management-system in cluster $1 created." + + # Once namespace is created, check if config-managment pods are ready + kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null + export exit_code=$? + + while [ ! " ${exit_code} " -eq 0 ] + do + sleep 5 + echo -e "Waiting for config-management pods in cluster $1 to become ready..." + kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null + export exit_code=$? + done + + echo -e "Config-management pods in cluster $1 are ready." + return +} + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +PROJECT_ID=$1 +CLUSTER_NAME=$2 +CLUSTER_LOCATION=$3 +USE_EXISTING_CONTEXT=$4 + +# Check if we need to use the current context +if [ -z ${USE_EXISTING_CONTEXT+x} ]; then + # GKE Cluster. Use the GKE cluster context + is_configsync_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" +else + echo "USE_EXISTING_CONTEXT variable is set. Using current context to wait for deployment to be ready." + # Get the current context. This can be used for non GKE Clusters + CURRENT_CONTEXT=$(kubectl config current-context) + is_configsync_ready "${CURRENT_CONTEXT}" +fi diff --git a/modules/k8s-operator-crd-support/variables.tf b/modules/k8s-operator-crd-support/variables.tf index 078cdc808..508c885fe 100644 --- a/modules/k8s-operator-crd-support/variables.tf +++ b/modules/k8s-operator-crd-support/variables.tf @@ -40,6 +40,12 @@ variable "operator_latest_manifest_url" { type = string } +variable "enable_multi_repo" { + description = "Whether to use Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)." + type = bool + default = false +} + variable "sync_repo" { description = "ACM Git repo address" type = string @@ -50,12 +56,24 @@ variable "secret_type" { type = string } +variable "secret_ref_name" { + description = "Name of Secret to use for authentication (Config Sync multi-repo setup only). If un-set, uses Config Management default." + type = string + default = "" +} + variable "sync_branch" { description = "ACM repo Git branch. If un-set, uses Config Management default." type = string default = "" } +variable "sync_revision" { + description = "ACM repo Git revision. If un-set, uses Config Management default." + type = string + default = "" +} + variable "policy_dir" { description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default." type = string @@ -105,6 +123,11 @@ variable "operator_cr_template_path" { type = string } +variable "rootsync_cr_template_path" { + description = "path to template file to use for the root sync definition (Config Sync multi-repo setup only)" + type = string +} + variable "source_format" { description = <