Skip to content

Commit

Permalink
feat!: Add multi-repo support for Config Sync (#872)
Browse files Browse the repository at this point in the history
* Add multi-repo support for Config Sync

* Run make build

* increase wait time for configsync api

* Update modules/k8s-operator-crd-support/main.tf

Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>

* Enhance wait_for_configsync_api step

Co-authored-by: Morgante Pell <morgante.pell@morgante.net>
Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>
Co-authored-by: Morgante Pell <morgantep@google.com>
  • Loading branch information
4 people committed May 17, 2021
1 parent 7531f90 commit 23da103
Show file tree
Hide file tree
Showing 12 changed files with 231 additions and 16 deletions.
4 changes: 3 additions & 1 deletion modules/acm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ By default, this module will attempt to download the ACM operator from Google di
| cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes |
| create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no |
| enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no |
| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no |
| enable\_policy\_controller | Whether to enable the ACM Policy Controller on the cluster | `bool` | `true` | no |
| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map` | `null` | no |
| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map(any)` | `null` | no |
| install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no |
| location | GCP location used to reach cluster. | `string` | n/a | yes |
| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no |
Expand All @@ -62,6 +63,7 @@ By default, this module will attempt to download the ACM operator from Google di
| ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |
| sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no |
| sync\_repo | ACM Git repo address | `string` | n/a | yes |
| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no |
| use\_existing\_context | Use existing kubecontext to auth kube-api. | `bool` | `false` | no |

## Outputs
Expand Down
4 changes: 4 additions & 0 deletions modules/acm/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,10 @@ module "acm_operator" {
project_id = var.project_id
location = var.location
operator_path = var.operator_path
enable_multi_repo = var.enable_multi_repo
sync_repo = var.sync_repo
sync_branch = var.sync_branch
sync_revision = var.sync_revision
policy_dir = var.policy_dir
cluster_endpoint = var.cluster_endpoint
create_ssh_key = var.create_ssh_key
Expand All @@ -56,4 +58,6 @@ module "acm_operator" {
operator_cr_template_path = "${path.module}/templates/acm-config.yml.tpl"
operator_credential_namespace = "config-management-system"
operator_credential_name = "git-creds"

rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl"
}
5 changes: 5 additions & 0 deletions modules/acm/templates/acm-config.yml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,15 @@ spec:
enabled: ${enable_policy_controller}
templateLibraryInstalled: ${install_template_library}
logDeniesEnabled: ${enable_log_denies}
%{ if enable_multi_repo ~}
enableMultiRepo: true
%{ else ~}
git:
syncRepo: ${sync_repo}
secretType: ${secret_type}
${policy_dir_node}
${sync_branch_node}
${sync_revision_node}
${source_format_node}
%{ endif ~}
${hierarchy_controller_map_node}
14 changes: 14 additions & 0 deletions modules/acm/templates/root-sync.yml.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: configsync.gke.io/v1beta1
kind: RootSync
metadata:
name: root-sync
namespace: config-management-system
spec:
${source_format_node}
git:
repo: ${sync_repo}
auth: ${secret_type}
${policy_dir_node}
${sync_branch_node}
${sync_revision_node}
${secret_ref_node}
14 changes: 13 additions & 1 deletion modules/acm/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ variable "operator_path" {
default = null
}

variable "enable_multi_repo" {
description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)."
type = bool
default = false
}

variable "sync_repo" {
description = "ACM Git repo address"
type = string
Expand All @@ -46,6 +52,12 @@ variable "sync_branch" {
default = ""
}

variable "sync_revision" {
description = "ACM repo Git revision. If un-set, uses Config Management default."
type = string
default = ""
}

variable "policy_dir" {
description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default."
type = string
Expand Down Expand Up @@ -95,7 +107,7 @@ variable "source_format" {

variable "hierarchy_controller" {
description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details"
type = map
type = map(any)
default = null
}

Expand Down
4 changes: 3 additions & 1 deletion modules/config-sync/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ To deploy this config:
| cluster\_endpoint | Kubernetes cluster endpoint. | `string` | n/a | yes |
| cluster\_name | GCP cluster name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes |
| create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no |
| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map` | `null` | no |
| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no |
| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map(any)` | `null` | no |
| location | GCP location used to reach cluster. | `string` | n/a | yes |
| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no |
| policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no |
Expand All @@ -60,6 +61,7 @@ To deploy this config:
| ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |
| sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no |
| sync\_repo | ACM Git repo address | `string` | n/a | yes |
| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no |

## Outputs

Expand Down
4 changes: 4 additions & 0 deletions modules/config-sync/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ module "configsync_operator" {
project_id = var.project_id
location = var.location
operator_path = var.operator_path
enable_multi_repo = var.enable_multi_repo
sync_repo = var.sync_repo
sync_branch = var.sync_branch
sync_revision = var.sync_revision
policy_dir = var.policy_dir
cluster_endpoint = var.cluster_endpoint
create_ssh_key = var.create_ssh_key
Expand All @@ -36,4 +38,6 @@ module "configsync_operator" {
operator_cr_template_path = "${path.module}/templates/config-sync-config.yml.tpl"
operator_credential_namespace = "config-management-system"
operator_credential_name = "git-creds"

rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl"
}
14 changes: 14 additions & 0 deletions modules/config-sync/templates/root-sync.yml.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: configsync.gke.io/v1beta1
kind: RootSync
metadata:
name: root-sync
namespace: config-management-system
spec:
${source_format_node}
git:
repo: ${sync_repo}
auth: ${secret_type}
${policy_dir_node}
${sync_branch_node}
${sync_revision_node}
${secret_ref_node}
14 changes: 13 additions & 1 deletion modules/config-sync/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ variable "operator_path" {
default = null
}

variable "enable_multi_repo" {
description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)."
type = bool
default = false
}

variable "sync_repo" {
description = "ACM Git repo address"
type = string
Expand All @@ -46,6 +52,12 @@ variable "sync_branch" {
default = ""
}

variable "sync_revision" {
description = "ACM repo Git revision. If un-set, uses Config Management default."
type = string
default = ""
}

variable "policy_dir" {
description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default."
type = string
Expand Down Expand Up @@ -82,6 +94,6 @@ variable "source_format" {

variable "hierarchy_controller" {
description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details."
type = map
type = map(any)
default = null
}
81 changes: 70 additions & 11 deletions modules/k8s-operator-crd-support/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,21 @@
*/

locals {
cluster_endpoint = "https://${var.cluster_endpoint}"
private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key
k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type
should_download_manifest = var.operator_path == null ? true : false
manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path
sync_branch_node = var.sync_branch != "" ? format("syncBranch: %s", var.sync_branch) : ""
policy_dir_node = var.policy_dir != "" ? format("policyDir: %s", var.policy_dir) : ""
hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchyController:\n %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")))
source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : ""
append_arg_use_existing_context_for_gatekeeper = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : ""
cluster_endpoint = "https://${var.cluster_endpoint}"
private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key
k8sop_creds_secret_key = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type
should_download_manifest = var.operator_path == null ? true : false
manifest_path = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path
sync_branch_property = var.enable_multi_repo ? "branch" : "syncBranch"
sync_branch_node = var.sync_branch != "" ? format("%s: %s", local.sync_branch_property, var.sync_branch) : ""
sync_revision_property = var.enable_multi_repo ? "revision" : "syncRev"
sync_revision_node = var.sync_revision != "" ? format("%s: %s", local.sync_revision_property, var.sync_revision) : ""
policy_dir_property = var.enable_multi_repo ? "dir" : "policyDir"
policy_dir_node = var.policy_dir != "" ? format("%s: %s", local.policy_dir_property, var.policy_dir) : ""
secret_ref_node = var.create_ssh_key == true || var.ssh_auth_key != null ? format("secretRef:\n name: %s", var.operator_credential_name) : ""
hierarchy_controller_map_node = var.hierarchy_controller == null ? "" : format("hierarchyController:\n %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")))
source_format_node = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : ""
append_arg_use_existing_context = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : ""
}

module "k8sop_manifest" {
Expand Down Expand Up @@ -82,8 +87,10 @@ data "template_file" "k8sop_config" {
template = file(var.operator_cr_template_path)
vars = {
cluster_name = var.cluster_name
enable_multi_repo = var.enable_multi_repo
sync_repo = var.sync_repo
sync_branch_node = local.sync_branch_node
sync_revision_node = local.sync_revision_node
policy_dir_node = local.policy_dir_node
secret_type = var.create_ssh_key ? "ssh" : var.secret_type
enable_policy_controller = var.enable_policy_controller ? "true" : "false"
Expand All @@ -109,6 +116,58 @@ module "k8sop_config" {
kubectl_destroy_command = "kubectl delete -f - <<EOF\n${data.template_file.k8sop_config.rendered}EOF"
}


data "template_file" "rootsync_config" {

template = file(var.rootsync_cr_template_path)
vars = {
sync_repo = var.sync_repo
sync_branch_node = local.sync_branch_node
sync_revision_node = local.sync_revision_node
policy_dir_node = local.policy_dir_node
secret_ref_node = local.secret_ref_node
secret_type = var.create_ssh_key ? "ssh" : var.secret_type
source_format_node = local.source_format_node
}
}

module "wait_for_configsync_api" {
source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
version = "~> 2.0.2"
enabled = var.enable_multi_repo

module_depends_on = [module.k8sop_config.wait]
cluster_name = var.cluster_name
project_id = var.project_id
cluster_location = var.location
create_cmd_triggers = {
rootsync = data.template_file.rootsync_config.rendered,
script_sha1 = sha1(file("${path.module}/scripts/wait_for_configsync.sh"))
}
service_account_key_file = var.service_account_key_file
use_existing_context = var.use_existing_context

kubectl_create_command = "${path.module}/scripts/wait_for_configsync.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context}"
kubectl_destroy_command = ""
}

module "rootsync_config" {
source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
version = "~> 2.0.2"
enabled = var.enable_multi_repo

module_depends_on = [module.wait_for_configsync_api.wait]
cluster_name = var.cluster_name
project_id = var.project_id
cluster_location = var.location
create_cmd_triggers = { rootsync = data.template_file.rootsync_config.rendered }
service_account_key_file = var.service_account_key_file
use_existing_context = var.use_existing_context

kubectl_create_command = "kubectl apply -f - <<EOF\n${data.template_file.rootsync_config.rendered}EOF"
kubectl_destroy_command = "kubectl delete -f - <<EOF\n${data.template_file.rootsync_config.rendered}EOF"
}

module "wait_for_gatekeeper" {
source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
version = "~> 2.0.2"
Expand All @@ -121,6 +180,6 @@ module "wait_for_gatekeeper" {
service_account_key_file = var.service_account_key_file
use_existing_context = var.use_existing_context

kubectl_create_command = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context_for_gatekeeper}"
kubectl_create_command = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context}"
kubectl_destroy_command = ""
}
64 changes: 64 additions & 0 deletions modules/k8s-operator-crd-support/scripts/wait_for_configsync.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

is_configsync_ready() {
# Check if config-management-system namespace exists
kubectl --context "$1" get namespace config-management-system &> /dev/null
export exit_code=$?
while [ ! " ${exit_code} " -eq 0 ]
do
sleep 5
echo -e "Waiting for namespace config-mangement-system in cluster $1 to be created..."
kubectl --context "$1" get namespace config-management-system &> /dev/null
export exit_code=$?
done
echo -e "Namespace config-management-system in cluster $1 created."

# Once namespace is created, check if config-managment pods are ready
kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null
export exit_code=$?

while [ ! " ${exit_code} " -eq 0 ]
do
sleep 5
echo -e "Waiting for config-management pods in cluster $1 to become ready..."
kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null
export exit_code=$?
done

echo -e "Config-management pods in cluster $1 are ready."
return
}

if [ "$#" -lt 3 ]; then
>&2 echo "Not all expected arguments set."
exit 1
fi

PROJECT_ID=$1
CLUSTER_NAME=$2
CLUSTER_LOCATION=$3
USE_EXISTING_CONTEXT=$4

# Check if we need to use the current context
if [ -z ${USE_EXISTING_CONTEXT+x} ]; then
# GKE Cluster. Use the GKE cluster context
is_configsync_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}"
else
echo "USE_EXISTING_CONTEXT variable is set. Using current context to wait for deployment to be ready."
# Get the current context. This can be used for non GKE Clusters
CURRENT_CONTEXT=$(kubectl config current-context)
is_configsync_ready "${CURRENT_CONTEXT}"
fi
Loading

0 comments on commit 23da103

Please sign in to comment.