diff --git a/.gitignore b/.gitignore index 899be3e74..611ad6dfd 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,7 @@ credentials.json # File to populate env vars used by Docker test runs .envrc + +# ignore generated ASM yamls in /workspace/test/fixtures/simple_regional_with_asm as it is a test +# in a production scenario these files are expected to be checked in +/test/fixtures/simple_regional_with_asm/asm-dir diff --git a/.kitchen.yml b/.kitchen.yml index 5796b039b..e24761646 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -228,3 +228,13 @@ suites: systems: - name: safer_cluster_iap_bastion backend: local + - name: "simple_regional_with_asm" + driver: + root_module_directory: test/fixtures/simple_regional_with_asm + verifier: + systems: + - name: simple_regional_with_asm + backend: local + controls: + - gcloud + - kubectl diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 27c9c79d9..905ed5489 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -404,6 +404,26 @@ steps: - verify safer-cluster-iap-bastion-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy safer-cluster-iap-bastion-local'] +- id: create simple-regional-with-asm-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-with-asm-local'] +- id: converge simple-regional-with-asm-local + waitFor: + - create simple-regional-with-asm-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-with-asm-local'] +- id: verify simple-regional-with-asm-local + waitFor: + - converge simple-regional-with-asm-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-with-asm-local'] +- id: destroy simple-regional-with-asm-local + waitFor: + - verify simple-regional-with-asm-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-asm-local'] tags: - 'ci' - 'integration' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 0f399c4e1..84fbf1532 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +timeout: 900s steps: - id: 'lint-tests' name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' diff --git a/examples/simple_regional_with_asm/README.md b/examples/simple_regional_with_asm/README.md new file mode 100644 index 000000000..8639f5989 --- /dev/null +++ b/examples/simple_regional_with_asm/README.md @@ -0,0 +1,44 @@ +# Simple Regional Cluster with ASM + +This example illustrates how to create a simple regional cluster with ASM. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| identity\_namespace | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The default service account used for running nodes. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_with_asm/main.tf b/examples/simple_regional_with_asm/main.tf new file mode 100644 index 000000000..907a6338f --- /dev/null +++ b/examples/simple_regional_with_asm/main.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-asm2" +} + +provider "google-beta" { + version = "~> 3.23.0" + region = var.region +} + +data "google_project" "project" { + project_id = var.project_id +} + +module "gke" { + source = "../../modules/beta-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + release_channel = "REGULAR" + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + network_policy = false + cluster_resource_labels = { "mesh_id" : "proj-${data.google_project.project.number}" } + node_pools = [ + { + name = "asm-node-pool" + autoscaling = false + # ASM requires minimum 4 nodes and e2-standard-4 + # As this is a regional cluster we have node_count * 3 = 6 nodes + node_count = 2 + machine_type = "e2-standard-4" + }, + ] +} + +module "asm" { + source = "../../modules/asm" + cluster_name = module.gke.name + cluster_endpoint = module.gke.endpoint + project_id = var.project_id + location = module.gke.location + use_tf_google_credentials_env_var = true +} + +data "google_client_config" "default" { +} diff --git a/examples/simple_regional_with_asm/outputs.tf b/examples/simple_regional_with_asm/outputs.tf new file mode 100644 index 000000000..0d770aa80 --- /dev/null +++ b/examples/simple_regional_with_asm/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} diff --git a/examples/simple_regional_with_asm/test_outputs.tf b/examples/simple_regional_with_asm/test_outputs.tf new file mode 100644 index 000000000..71e5965e0 --- /dev/null +++ b/examples/simple_regional_with_asm/test_outputs.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.gke.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network" { + value = var.network +} + +output "subnetwork" { + value = var.subnetwork +} + +output "location" { + value = module.gke.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = var.ip_range_pods +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = var.ip_range_services +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.gke.master_version +} + +output "identity_namespace" { + value = module.gke.identity_namespace +} diff --git a/examples/simple_regional_with_asm/variables.tf b/examples/simple_regional_with_asm/variables.tf new file mode 100644 index 000000000..22ee4e456 --- /dev/null +++ b/examples/simple_regional_with_asm/variables.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for services" +} diff --git a/examples/simple_regional_with_asm/versions.tf b/examples/simple_regional_with_asm/versions.tf new file mode 100644 index 000000000..832ec1df3 --- /dev/null +++ b/examples/simple_regional_with_asm/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/asm/README.md b/modules/asm/README.md new file mode 100644 index 000000000..1c0156162 --- /dev/null +++ b/modules/asm/README.md @@ -0,0 +1,54 @@ +# Terraform Kubernetes Engine ASM Submodule + +This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes cluster. + +Specifically, this module automates the following steps for [installing ASM](https://cloud.google.com/service-mesh/docs/install): + +1. Installing the ASM Istio Operator on your cluster. +2. Optionally registering your cluster with GKE Hub. + +## Usage + +There is a [full example](../../examples/simple_regional_with_asm) provided. Simple usage is as follows: + +```tf +module "asm" { + source = "terraform-google-modules/kubernetes-engine/google//modules/asm" + + project_id = "my-project-id" + cluster_name = "my-cluster-name" + location = module.gke.location + cluster_endpoint = module.gke.endpoint +} +``` + +To deploy this config: +1. Run `terraform apply` + +## Requirements + +- Anthos Service Mesh [requires](https://cloud.google.com/service-mesh/docs/gke-install-existing-cluster#requirements) an active Anthos license. +- GKE cluster must have minimum four nodes. +- Minimum machine type is `e2-standard-4`. +- GKE cluster must be enrolled in a release channel. ASM does not support static version. +- ASM on a private GKE cluster requires adding a firewall rule to open port 15017 if you want to use [automatic sidecar injection](https://cloud.google.com/service-mesh/docs/proxy-injection). +- Only one ASM per Google Cloud project is supported. + + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_endpoint | The GKE cluster endpoint. | string | n/a | yes | +| cluster\_name | The unique name to identify the cluster in ASM. | string | n/a | yes | +| enable\_gke\_hub\_registration | Enables GKE Hub Registration when set to true | bool | `"true"` | no | +| gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | string | `"296.0.1"` | no | +| gke\_hub\_membership\_name | Memebership name that uniquely represents the cluster being registered on the Hub | string | `"gke-asm-membership"` | no | +| gke\_hub\_sa\_name | Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace. | string | `"gke-hub-sa"` | no | +| location | The location (zone or region) this cluster has been created in. | string | n/a | yes | +| project\_id | The project in which the resource belongs. | string | n/a | yes | +| skip\_gcloud\_download | Whether to skip downloading gcloud (assumes gcloud and kubectl already available outside the module) | bool | `"true"` | no | +| use\_tf\_google\_credentials\_env\_var | Optional GOOGLE_CREDENTIALS environment variable to be activated. | bool | `"false"` | no | + + diff --git a/modules/asm/main.tf b/modules/asm/main.tf new file mode 100644 index 000000000..06f5ce276 --- /dev/null +++ b/modules/asm/main.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gke_hub_sa_key = var.enable_gke_hub_registration ? google_service_account_key.gke_hub_key[0].private_key : "" +} + +data "google_container_cluster" "primary" { + name = var.cluster_name + project = var.project_id + location = var.location +} + +data "google_client_config" "default" { +} + +module "asm_install" { + source = "terraform-google-modules/gcloud/google" + version = "~> 1.0" + module_depends_on = [var.cluster_endpoint] + + platform = "linux" + gcloud_sdk_version = var.gcloud_sdk_version + skip_download = var.skip_gcloud_download + upgrade = true + use_tf_google_credentials_env_var = var.use_tf_google_credentials_env_var + additional_components = ["kubectl", "kpt"] + + create_cmd_entrypoint = "${path.module}/scripts/install_asm.sh" + create_cmd_body = "${var.project_id} ${var.cluster_name} ${var.location}" + destroy_cmd_entrypoint = "${path.module}/scripts/kubectl_wrapper.sh" + destroy_cmd_body = "https://${var.cluster_endpoint} ${data.google_client_config.default.access_token} ${data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate} kubectl delete ns istio-system" +} + +resource "google_service_account" "gke_hub_sa" { + count = var.enable_gke_hub_registration ? 1 : 0 + account_id = var.gke_hub_sa_name + project = var.project_id + display_name = "Service Account for GKE Hub Registration" +} + +resource "google_project_iam_member" "gke_hub_member" { + count = var.enable_gke_hub_registration ? 1 : 0 + project = var.project_id + role = "roles/gkehub.connect" + member = "serviceAccount:${google_service_account.gke_hub_sa[0].email}" +} + +resource "google_service_account_key" "gke_hub_key" { + count = var.enable_gke_hub_registration ? 1 : 0 + service_account_id = google_service_account.gke_hub_sa[0].name +} + +module "gke_hub_registration" { + source = "terraform-google-modules/gcloud/google" + version = "~> 1.0" + + platform = "linux" + gcloud_sdk_version = var.gcloud_sdk_version + skip_download = var.skip_gcloud_download + upgrade = true + enabled = var.enable_gke_hub_registration + use_tf_google_credentials_env_var = var.use_tf_google_credentials_env_var + module_depends_on = [module.asm_install.wait] + + create_cmd_entrypoint = "${path.module}/scripts/gke_hub_registration.sh" + create_cmd_body = "${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key}" + destroy_cmd_entrypoint = "gcloud" + destroy_cmd_body = "container hub memberships unregister ${var.gke_hub_membership_name} --gke-cluster=${var.location}/${var.cluster_name} --project ${var.project_id}" +} diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf new file mode 100644 index 000000000..cfccff84c --- /dev/null +++ b/modules/asm/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/modules/asm/scripts/gke_hub_registration.sh b/modules/asm/scripts/gke_hub_registration.sh new file mode 100755 index 000000000..84757051a --- /dev/null +++ b/modules/asm/scripts/gke_hub_registration.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ "$#" -lt 4 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +MEMBERSHIP_NAME=$1 +CLUSTER_LOCATION=$2 +CLUSTER_NAME=$3 +SERVICE_ACCOUNT_KEY=$4 + +#write temp key, cleanup at exit +tmp_file=$(mktemp) +# shellcheck disable=SC2064 +trap "rm -rf $tmp_file" EXIT +echo "${SERVICE_ACCOUNT_KEY}" | base64 --decode > "$tmp_file" + +gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --service-account-key-file="${tmp_file}" --quiet diff --git a/modules/asm/scripts/install_asm.sh b/modules/asm/scripts/install_asm.sh new file mode 100755 index 000000000..6b8feea4b --- /dev/null +++ b/modules/asm/scripts/install_asm.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +PROJECT_ID=$1 +CLUSTER_NAME=$2 +CLUSTER_LOCATION=$3 +ASM_RESOURCES="asm-dir" +BASE_DIR="asm-base-dir" +# check for needed binaries +# kustomize is a requirement for installing ASM and is not available via gcloud. Safely exit if not available. +if [[ -z $(command -v kustomize) ]]; then + echo "kustomize is unavailable. Skipping ASM installation. Please install kustomize, add to PATH and rerun terraform apply." + exit 1 +fi +# # check docker which is optionally used for validating asm yaml using gcr.io/kustomize-functions/validate-asm:v0.1.0 +# if [[ $(command -v docker) ]]; then +# echo "Docker is available. ASM yaml validation will be performed." +# else +# echo "ASM yaml validation will be skipped as Docker is unavailable" +# SKIP_ASM_VALIDATION=true +# fi +mkdir -p $ASM_RESOURCES +pushd $ASM_RESOURCES +gcloud config set project "${PROJECT_ID}" +if [[ -d ./asm-patch ]]; then + echo "ASM patch directory exists. Skipping download..." +else + echo "Downloading ASM patch" + kpt pkg get https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages.git/asm-patch@release-1.5-asm . +fi +gcloud beta anthos export "${CLUSTER_NAME}" --output-directory ${BASE_DIR} --project "${PROJECT_ID}" --location "${CLUSTER_LOCATION}" +kpt cfg set asm-patch/ base-dir ../${BASE_DIR} +kpt cfg set asm-patch/ gcloud.core.project "${PROJECT_ID}" +kpt cfg set asm-patch/ gcloud.container.cluster "${CLUSTER_NAME}" +kpt cfg set asm-patch/ gcloud.compute.location "${CLUSTER_LOCATION}" +kpt cfg list-setters asm-patch/ +pushd ${BASE_DIR} && kustomize create --autodetect --namespace "${PROJECT_ID}" && popd +pushd asm-patch && kustomize build -o ../${BASE_DIR}/all.yaml && popd +# # skip validate as we should investigate if we can check this without having to resort to dind +# if [[ ${SKIP_ASM_VALIDATION} ]]; then +# echo "Skipping ASM validation..." +# else +# echo "Running ASM validation..." +# kpt fn source ${BASE_DIR} | kpt fn run --image gcr.io/kustomize-functions/validate-asm:v0.1.0 +# fi +gcloud beta anthos apply ${BASE_DIR} +kubectl wait --for=condition=available --timeout=600s deployment --all -n istio-system diff --git a/modules/asm/scripts/kubectl_wrapper.sh b/modules/asm/scripts/kubectl_wrapper.sh new file mode 100755 index 000000000..e92300bcb --- /dev/null +++ b/modules/asm/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf new file mode 100644 index 000000000..f0d358051 --- /dev/null +++ b/modules/asm/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cluster_name" { + description = "The unique name to identify the cluster in ASM." + type = string +} + +variable "cluster_endpoint" { + description = "The GKE cluster endpoint." + type = string +} + +variable "project_id" { + description = "The project in which the resource belongs." + type = string +} + +variable "location" { + description = "The location (zone or region) this cluster has been created in." + type = string +} + +variable "skip_gcloud_download" { + description = "Whether to skip downloading gcloud (assumes gcloud and kubectl already available outside the module)" + type = bool + default = true +} + +variable "use_tf_google_credentials_env_var" { + description = "Optional GOOGLE_CREDENTIALS environment variable to be activated." + type = bool + default = false +} + +variable "gcloud_sdk_version" { + description = "The gcloud sdk version to use. Minimum required version is 293.0.0" + type = string + default = "296.0.1" +} + +variable "enable_gke_hub_registration" { + description = "Enables GKE Hub Registration when set to true" + type = bool + default = true +} + +variable "gke_hub_sa_name" { + description = "Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace." + type = string + default = "gke-hub-sa" +} + +variable "gke_hub_membership_name" { + description = "Memebership name that uniquely represents the cluster being registered on the Hub" + type = string + default = "gke-asm-membership" +} diff --git a/test/fixtures/simple_regional_with_asm/example.tf b/test/fixtures/simple_regional_with_asm/example.tf new file mode 100644 index 000000000..d42c5097d --- /dev/null +++ b/test/fixtures/simple_regional_with_asm/example.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project_ids[2] +} + +module "example" { + source = "../../../examples/simple_regional_with_asm" + + project_id = var.project_ids[2] + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/simple_regional_with_asm/network.tf b/test/fixtures/simple_regional_with_asm/network.tf new file mode 100644 index 000000000..194b20884 --- /dev/null +++ b/test/fixtures/simple_regional_with_asm/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + version = "~> 3.16.0" + project = var.project_ids[2] +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} diff --git a/test/fixtures/simple_regional_with_asm/outputs.tf b/test/fixtures/simple_regional_with_asm/outputs.tf new file mode 100644 index 000000000..515bac632 --- /dev/null +++ b/test/fixtures/simple_regional_with_asm/outputs.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.example.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.example.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} + +output "project_number" { + description = "The project number of the ASM project" + value = data.google_project.project.number +} diff --git a/test/fixtures/simple_regional_with_asm/variables.tf b/test/fixtures/simple_regional_with_asm/variables.tf new file mode 100644 index 000000000..16f1b9367 --- /dev/null +++ b/test/fixtures/simple_regional_with_asm/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} + +variable "zones" { + type = list(string) + description = "The GCP zones to create and test resources in, for applicable tests" + default = ["us-central1-a", "us-central1-b", "us-central1-c"] +} diff --git a/test/integration/simple_regional_with_asm/controls/gcloud.rb b/test/integration/simple_regional_with_asm/controls/gcloud.rb new file mode 100644 index 000000000..1b97dfe68 --- /dev/null +++ b/test/integration/simple_regional_with_asm/controls/gcloud.rb @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +project_number = attribute('project_number') +location = attribute('location') +cluster_name = attribute('cluster_name') +k8s_service_account_email = attribute('k8s_service_account_email') +k8s_service_account_name = attribute('k8s_service_account_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud beta --project=#{project_id} container clusters --region=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "mesh id" do + it "is correct" do + expect(data['resourceLabels']["mesh_id"]).to eq "proj-#{project_number}" + end + end + end + + describe command("gcloud container hub memberships describe gke-asm-membership --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:hub) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + it "membership has expected gke cluster" do + expect(hub['endpoint']["gkeCluster"]["resourceLink"]).to eq "//container.googleapis.com/projects/#{project_id}/locations/#{location}/clusters/#{cluster_name}" + end + end +end diff --git a/test/integration/simple_regional_with_asm/controls/kubectl.rb b/test/integration/simple_regional_with_asm/controls/kubectl.rb new file mode 100644 index 000000000..e0b42a317 --- /dev/null +++ b/test/integration/simple_regional_with_asm/controls/kubectl.rb @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'kubeclient' +require 'rest-client' + +require 'base64' + +kubernetes_endpoint = attribute('kubernetes_endpoint') +client_token = attribute('client_token') +ca_certificate = attribute('ca_certificate') + +control "kubectl" do + title "Kubernetes configuration" + + describe "kubernetes" do + let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } + let(:client) do + cert_store = OpenSSL::X509::Store.new + cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) + Kubeclient::Client.new( + kubernetes_http_endpoint, + "v1", + ssl_options: { + cert_store: cert_store, + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + }, + auth_options: { + bearer_token: Base64.decode64(client_token), + }, + ) + end + + describe "Mesh" do + describe "CA" do + let(:pod) { client.get_pods(label_selector:"app=istio-ingressgateway", namespace: "istio-system", as: :raw) } + it "ingressgateway exists" do + expect(pod).not_to be_nil + end + + it "ingressgateway has correct CA_ADDR " do + expect(pod).to include("{\"name\":\"CA_ADDR\",\"value\":\"meshca.googleapis.com:443\"}") + end + end + end + end +end diff --git a/test/integration/simple_regional_with_asm/inspec.yml b/test/integration/simple_regional_with_asm/inspec.yml new file mode 100644 index 000000000..d30ad0c64 --- /dev/null +++ b/test/integration/simple_regional_with_asm/inspec.yml @@ -0,0 +1,29 @@ +name: simple_regional_with_asm +attributes: + - name: cluster_name + required: true + type: string + - name: location + required: true + type: string + - name: project_id + required: true + type: string + - name: project_number + required: true + type: string + - name: k8s_service_account_email + required: true + type: string + - name: k8s_service_account_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: ca_certificate + required: true + type: string diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 6a85ba2b3..0a7fa49c1 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -34,9 +34,17 @@ locals { "roles/iam.roleAdmin", "roles/iap.admin", ] + # roles as documented https://cloud.google.com/service-mesh/docs/gke-install-new-cluster#setting_up_your_project + int_asm_required_roles = [ + "roles/editor", + "roles/container.admin", + "roles/resourcemanager.projectIamAdmin", + "roles/iam.serviceAccountAdmin", + "roles/iam.serviceAccountKeyAdmin", + "roles/gkehub.admin", + ] } - resource "random_id" "random_suffix" { byte_length = 2 } @@ -59,6 +67,12 @@ resource "google_service_account" "gke_sa_2" { display_name = "gke-sa-int-test-p2" } +resource "google_service_account" "gke_sa_asm" { + project = module.gke-project-asm.project_id + account_id = "gke-sa-int-test-asm-${random_id.random_suffix.hex}" + display_name = "gke-sa-int-test-asm" +} + resource "google_project_iam_member" "int_test_1" { count = length(local.int_required_roles) @@ -75,6 +89,14 @@ resource "google_project_iam_member" "int_test_2" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "int_test_asm" { + for_each = toset(concat(local.int_required_roles, local.int_asm_required_roles)) + + project = module.gke-project-asm.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_service_account_key" "int_test" { service_account_id = google_service_account.int_test.id } diff --git a/test/setup/main.tf b/test/setup/main.tf index 9aa9221bf..74cf51e29 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -69,3 +69,30 @@ module "gke-project-2" { "storage-api.googleapis.com", ] } + +# apis as documented https://cloud.google.com/service-mesh/docs/gke-install-new-cluster#setting_up_your_project +module "gke-project-asm" { + source = "terraform-google-modules/project-factory/google" + version = "~> 3.0" + + name = "ci-gke-asm" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + activate_apis = [ + "container.googleapis.com", + "compute.googleapis.com", + "monitoring.googleapis.com", + "logging.googleapis.com", + "meshca.googleapis.com", + "meshtelemetry.googleapis.com", + "meshconfig.googleapis.com", + "iamcredentials.googleapis.com", + "anthos.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", + "cloudresourcemanager.googleapis.com", + ] +} diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 1365f3045..8b85098fc 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -15,7 +15,7 @@ */ output "project_ids" { - value = [module.gke-project-1.project_id, module.gke-project-2.project_id] + value = [module.gke-project-1.project_id, module.gke-project-2.project_id, module.gke-project-asm.project_id] } output "sa_key" { @@ -28,7 +28,7 @@ output "int_sa" { } output "compute_engine_service_accounts" { - value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email] + value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email, google_service_account.gke_sa_asm.email] } output "registry_project_id" {