From e8688fce06b3a2aedb7f9df34b96d9f2eae4e7c5 Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 01/15] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..ac5726feeb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -61,6 +61,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From 599dd2e3d801a116c8d44c2633c3425ff0d0b78b Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 02/15] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..ac5726feeb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -61,6 +61,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From b3a6d3e2d4bbba0d978e85591299e279cc7e9cad Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:01:16 -0600 Subject: [PATCH 03/15] add shielded vms, use new CI, address comments --- examples/safer_cluster/main.tf | 34 ++++++++++++------- examples/safer_cluster/variables.tf | 8 +++++ modules/safer-cluster/main.tf | 18 +++------- modules/safer-cluster/variables.tf | 7 ++++ test/fixtures/safer_cluster/example.tf | 15 ++++---- .../safer_cluster/controls/gcloud.rb | 12 +++++++ 6 files changed, 62 insertions(+), 32 deletions(-) diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 7b16e90abc..102d539502 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" + version = "~> 2.18.0" region = var.region } @@ -30,17 +30,27 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/safer-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - master_ipv4_cidr_block = "172.16.0.0/28" - + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + compute_engine_service_account = var.compute_engine_service_account + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] istio = var.istio cloudrun = var.cloudrun } diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 733103bc31..deb08f59e9 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -53,3 +53,11 @@ variable "cloudrun" { default = true } +variable "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation to use for the hosted master network" + default = "172.16.0.0/28" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 1233afa65c..9069e22559 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -45,9 +45,6 @@ module "gke" { horizontal_pod_autoscaling = var.horizontal_pod_autoscaling http_load_balancing = var.http_load_balancing - // Disable the dashboard. It creates risk by running as a very sensitive user. - kubernetes_dashboard = false - // We suggest the use coarse network policies to enforce restrictions in the // communication between pods. // @@ -55,8 +52,7 @@ module "gke" { // NetworkPolicies need to be configured in every namespace. The network // policies should be under the control of a cental cluster management team, // rather than individual teams. - network_policy = true - network_policy_provider = "CALICO" + network_policy = true maintenance_start_time = var.maintenance_start_time @@ -66,8 +62,6 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - disable_legacy_metadata_endpoints = true - node_pools = var.node_pools node_pools_labels = var.node_pools_labels @@ -78,7 +72,7 @@ module "gke" { node_pools_tags = var.node_pools_tags // TODO(mmontan): we generally considered applying - // just the cloud-platofrm scope and use Cloud IAM + // just the cloud-platoform scope and use Cloud IAM // If we have Workload Identity, are there advantages // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes @@ -86,9 +80,6 @@ module "gke" { stub_domains = var.stub_domains upstream_nameservers = var.upstream_nameservers - // We should use IP Alias. - configure_ip_masq = false - logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -101,7 +92,7 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + create_service_account = var.compute_engine_service_account == "" ? false : true service_account = var.compute_engine_service_account // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, @@ -148,7 +139,6 @@ module "gke" { }] resource_usage_export_dataset_id = var.resource_usage_export_dataset_id - node_metadata = "SECURE" // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). // Sandbox can also provide increased protection in other cases, at some performance cost. @@ -163,4 +153,6 @@ module "gke" { identity_namespace = "${var.project_id}.svc.id.goog" authenticator_security_group = var.authenticator_security_group + + enable_shielded_nodes = var.enable_shielded_nodes } diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 3a3ffefca2..71b4ba88ef 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -293,3 +293,10 @@ variable "compute_engine_service_account" { description = "Use the given service account for nodes rather than creating a new dedicated service account." default = "" } + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster." + default = true +} + diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index 87285c7416..c9d659ba67 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,11 +17,12 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" - region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index e9a78d7f5e..920664593b 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -174,6 +174,18 @@ ) ) end + + it "has shielded nodes" do + expect(node_pools).to include( + including( + "config" => including( + "shieldedInstanceConfig" => including( + "enableIntegrityMonitoring" => true, + ) + ), + ) + ) + end end end end From 31f833b078863ced726d74ffaa53a875dd9b47a1 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:16:47 -0600 Subject: [PATCH 04/15] adds docs from PR281, add desc --- examples/safer_cluster/README.md | 16 +- examples/safer_cluster/outputs.tf | 13 +- modules/safer-cluster/README.md | 275 +++++++++++++++++- .../safer_cluster/controls/gcloud.rb | 4 +- 4 files changed, 287 insertions(+), 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 7cd54188f4..93a7d4a1a7 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -1,9 +1,8 @@ # Safer GKE Cluster -This example illustrates how to instantiate the opinionanted Safer Cluster module. - -[^]: (autogen_docs_start) +This example illustrates how to instantiate the opinionated Safer Cluster module. + ## Inputs | Name | Description | Type | Default | Required | @@ -11,10 +10,10 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | @@ -24,13 +23,12 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | Name | Description | |------|-------------| -| ca\_certificate | | -| client\_token | | +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| credentials\_path | | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | | +| kubernetes\_endpoint | The cluster endpoint | | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | @@ -40,7 +38,7 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 0d972dcd88..2ff9f4734b 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -15,17 +15,20 @@ */ output "kubernetes_endpoint" { - sensitive = true - value = module.gke.endpoint + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint } output "client_token" { - sensitive = true - value = base64encode(data.google_client_config.default.access_token) + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) } output "ca_certificate" { - value = module.gke.ca_certificate + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate } output "service_account" { diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 6f0206d826..ad773982be 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,12 +1,277 @@ -# Safer Beta Cluster +# Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -The module defines a safer configuration for a GKE cluster. +[TOC] -TODO(mmontan): add documentation for the module. +This module defines an opinionated setup of GKE +cluster. We outline project configurations, cluster settings, and basic K8s +objects that permit a safer-than-default configuration. -[^]: (autogen_docs_start) +## Module Usage -[^]: (autogen_docs_end) +The module fixes a set of parameters to values suggested in the +[GKE harderning guide](https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster), +the CIS framework, and other best practices. + +The motivation for each setting, and its relation to harderning guides or other recommendations +is outline in `main.tf` as comments over individual settings. When security-relevant settings +are available for configuration, recommendations on their settings are documented in the `variables.tf` file. + +## Project Setup and Cloud IAM policy for GKE + +### Applications and Clusters + +- Different applications that access data with different sensitivity and do + not need to communicate with each other (e.g., dev and prod instances of the + same application) should be placed in different clusters. + + - This approach will limit the blast radius of errors. An security problem + in dev shouldn't impact production data. + +- If applications need to communicate (e.g., a frontend system calling + a backend), we suggest placing the two applications in the same cluster, in + different namespaces. + + - Placing them in the same cluster will provide fast network + communication, and the different namespaces will be configured to + provide some administrative isolation. Istio will be used to encrypt and + control communication between applications. + +- We suggest to store user or business data persistently in managed storage + services that are inventories and controlled by centralized teams. + (e.g., GCP storage services within a GCP organization). + + - Storing user or business data securely requires satisfying a large set of + requirements, such as data inventory, which might be harder to satisfy at + scale when data is stored opaquely within a cluster. Services like Cloud + Asset Inventory provide centralized teams ability to enumerate data stores. + +### Project Setup + +We suggest a GKE setup composed of multiple projects to separate responsabilities +between cluster operators, which need to administer the cluster; and product +developers, which mostly just want to deploy and debug applications. + +- *Cluster Projects (`project_id`):* GKE clusters storing sensitive data should have their + own projects, so that they can be administered independently (e.g., dev cluster; + production clusters; staging clusters should go in different projects.) + +- *A shared GCR project (`registry_project_id`):* all clusters can share the same GCR project. + + - Easier to share images between environments. The same image could be + progressively rolled-out in dev, staging, and then production. + - Easier to manage service account permissions: GCR requires authorizing + service accounts to access certain buckets, which are created only after + images are published. When the only service run by the project is GCR, + we can safely give project-wide read permissions to all buckets. + +- (optional) *Data Projects:* When the same cluster is shared by different + applications managed by different teams, we suggest separating the data for + each application by placing storage resources for each team in different + projects (e.g., a Spanner instance for application A in one project, GCS + bucket for application B in a different project). + + - This permits to control administrative access to the data more tightly, + as Cloud IAM policies for accessing the data can be managed by each + application team, rather than the team managing the cluster + infrastructure. + +Exception to such a setup: + +- When not using Shared VPCs, resources that require direct network connectivity + (e.g., a Cloud SQL instance), need to be placed in the same VPC (hence, project) + as the clusters from which connections are made. + +### Google Service Accounts + +We use GKE Workload Identity (BETA) to associate a GCP identity to each workload, +and limit the permissions associated with the cluster nodes. + +The Safer Cluster setup relies on several service accounts: + +- The module generates a service account to run nodes. Such a service account + has only permissions of sending logging data, metrics, and downloading containers + from the given GCR project. The following settings in the module will create + a service account with the above properties: + +``` +create_service_account = true +registry_project_id = +grant_registry_access = true +``` + +- A service account *for each application* running on the cluster (e.g., + `myproduct-sa@myproduct-prod.iam.gserviceaccount.com`). These service + accounts should be associated to the permissions required for running the + application, such as access to databases. + +``` +- email: myproduct + displayName: Google Service Account for containers running in the myproduct k8s namespace + policy: + # GKE workload identity authorization. This authorizes the Kubernetes Service Account + # myproduct/default from this project's identity namespace to impersonate the service account. + # https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + bindings: + - members: + - serviceAccount:product-prod.svc.id.goog[myproduct/default] + role: roles/iam.workloadIdentityUser +``` + +We suggest running different applications in different namespaces within the cluster. Each namespace +should be assigned to its own GCP service account to better define the Cloud IAM permissions required +for the application. + +If you are using more than 2 projects in your setup, you can consider creting +the service account in a different project to keep application and +infrastructure separate. For example, service accounts could be created in each team's project, +while the cluster runs in a centrally controlled project. + +
+*Why?* + +Separating the permissions associated with the infrastructure GKE nodes and the +application provides a simpler way to scale up the cluster: multiple applications +could be run in the same cluster, and each of them can run with tailored permissions +that limit the impact of compromises. + +Such a separation of identities is enabled by a GKE feature called Workload +Identity. The feature provides additional advantages such as a better protection +of the node's metadata server against attackers. + +
+ +### Cloud IAM Permissions for the GKE Cluster + +We suggest to mainly rely on Kubernetes RBAC to manage access control, and use +Cloud IAM to give users only the ability of configuring `kubectl` credentials. + +Engineers operating applications on the cluster should only be assigned the +Cloud IAM permission `roles/container.clusterViewer`. This role allows them to +obtain credentials for the cluster, but provides no further access to the +cluster objects. All cluster objects are protected by RBAC configurations, +defined below. + +
+*Why?* + +Both Cloud IAM and RBAC can be used to control access to GKE clusters. Those two +systems are combined as a "OR": an action is authorized if the necessary +permissions are provided by either RBAC _OR_ Cloud IAM + +However, Cloud IAM permissions are defined for a project: user get assigned the +same permissions over all clusters and all namespaces within each cluster. Such +a setup makes it hard to separate responsibilities between teams in charge of +managing clusters, and teams in charge of products. + +By relying on RBAC instead of Cloud IAM, we have a finer-grained control of the +permissions provided to engineers, and permits to restrict permissions to only +certain namespaces. + +
+ +You can add the following binding to the `myproduct-prod` project. + +``` +- members: + role: roles/container.clusterViewer` + - group: + - group: +``` + +The permissions won't allow engineers to SSH into nodes as part of the regular +development workflow. Such permissions should be granted only to the cluster +team, and used only in case of emergency. + +While RBAC permissions should be sufficient for most cases, we also suggest to +create an emergency superuser role that can be used, given a proper +justification, for resolving cases where regular permissions are insufficient. +For simplicity, we suggest using `roles/container.admin` and +`roles/compute.admin`, until more narrow roles can be defined given your usage. + +``` +- members: + role: roles/container.admin + - group: +- members: + role: roles/compute.admin + - group: +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | string | `""` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index 920664593b..c2cdd7bca0 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -1,10 +1,10 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From 2c1aa19fb53ae134b6be2ebb0e3a5b4200d4911f Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 05/15] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 39faa2e1e3..3d7be4b20b 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -45,6 +45,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From b41137159403d177ebe6427feb98be44950f65ac Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:01:16 -0600 Subject: [PATCH 06/15] add shielded vms, use new CI, address comments --- examples/safer_cluster/main.tf | 34 ++++++++++++------- examples/safer_cluster/variables.tf | 8 +++++ modules/safer-cluster/main.tf | 18 +++------- modules/safer-cluster/variables.tf | 7 ++++ test/fixtures/safer_cluster/example.tf | 15 ++++---- .../safer_cluster/controls/gcloud.rb | 12 +++++++ 6 files changed, 62 insertions(+), 32 deletions(-) diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 7b16e90abc..102d539502 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" + version = "~> 2.18.0" region = var.region } @@ -30,17 +30,27 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/safer-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - master_ipv4_cidr_block = "172.16.0.0/28" - + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + compute_engine_service_account = var.compute_engine_service_account + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] istio = var.istio cloudrun = var.cloudrun } diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 733103bc31..deb08f59e9 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -53,3 +53,11 @@ variable "cloudrun" { default = true } +variable "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation to use for the hosted master network" + default = "172.16.0.0/28" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 1233afa65c..9069e22559 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -45,9 +45,6 @@ module "gke" { horizontal_pod_autoscaling = var.horizontal_pod_autoscaling http_load_balancing = var.http_load_balancing - // Disable the dashboard. It creates risk by running as a very sensitive user. - kubernetes_dashboard = false - // We suggest the use coarse network policies to enforce restrictions in the // communication between pods. // @@ -55,8 +52,7 @@ module "gke" { // NetworkPolicies need to be configured in every namespace. The network // policies should be under the control of a cental cluster management team, // rather than individual teams. - network_policy = true - network_policy_provider = "CALICO" + network_policy = true maintenance_start_time = var.maintenance_start_time @@ -66,8 +62,6 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - disable_legacy_metadata_endpoints = true - node_pools = var.node_pools node_pools_labels = var.node_pools_labels @@ -78,7 +72,7 @@ module "gke" { node_pools_tags = var.node_pools_tags // TODO(mmontan): we generally considered applying - // just the cloud-platofrm scope and use Cloud IAM + // just the cloud-platoform scope and use Cloud IAM // If we have Workload Identity, are there advantages // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes @@ -86,9 +80,6 @@ module "gke" { stub_domains = var.stub_domains upstream_nameservers = var.upstream_nameservers - // We should use IP Alias. - configure_ip_masq = false - logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -101,7 +92,7 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + create_service_account = var.compute_engine_service_account == "" ? false : true service_account = var.compute_engine_service_account // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, @@ -148,7 +139,6 @@ module "gke" { }] resource_usage_export_dataset_id = var.resource_usage_export_dataset_id - node_metadata = "SECURE" // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). // Sandbox can also provide increased protection in other cases, at some performance cost. @@ -163,4 +153,6 @@ module "gke" { identity_namespace = "${var.project_id}.svc.id.goog" authenticator_security_group = var.authenticator_security_group + + enable_shielded_nodes = var.enable_shielded_nodes } diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 3a3ffefca2..71b4ba88ef 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -293,3 +293,10 @@ variable "compute_engine_service_account" { description = "Use the given service account for nodes rather than creating a new dedicated service account." default = "" } + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster." + default = true +} + diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index 87285c7416..c9d659ba67 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,11 +17,12 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" - region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index e9a78d7f5e..920664593b 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -174,6 +174,18 @@ ) ) end + + it "has shielded nodes" do + expect(node_pools).to include( + including( + "config" => including( + "shieldedInstanceConfig" => including( + "enableIntegrityMonitoring" => true, + ) + ), + ) + ) + end end end end From 8539316fe1708ed0555c43f64cbd0f59f26f6ba8 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:16:47 -0600 Subject: [PATCH 07/15] adds docs from PR281, add desc --- examples/safer_cluster/README.md | 16 +- examples/safer_cluster/outputs.tf | 13 +- modules/safer-cluster/README.md | 275 +++++++++++++++++- .../safer_cluster/controls/gcloud.rb | 4 +- 4 files changed, 287 insertions(+), 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 7cd54188f4..93a7d4a1a7 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -1,9 +1,8 @@ # Safer GKE Cluster -This example illustrates how to instantiate the opinionanted Safer Cluster module. - -[^]: (autogen_docs_start) +This example illustrates how to instantiate the opinionated Safer Cluster module. + ## Inputs | Name | Description | Type | Default | Required | @@ -11,10 +10,10 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | @@ -24,13 +23,12 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | Name | Description | |------|-------------| -| ca\_certificate | | -| client\_token | | +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| credentials\_path | | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | | +| kubernetes\_endpoint | The cluster endpoint | | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | @@ -40,7 +38,7 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 0d972dcd88..2ff9f4734b 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -15,17 +15,20 @@ */ output "kubernetes_endpoint" { - sensitive = true - value = module.gke.endpoint + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint } output "client_token" { - sensitive = true - value = base64encode(data.google_client_config.default.access_token) + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) } output "ca_certificate" { - value = module.gke.ca_certificate + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate } output "service_account" { diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 6f0206d826..ad773982be 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,12 +1,277 @@ -# Safer Beta Cluster +# Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -The module defines a safer configuration for a GKE cluster. +[TOC] -TODO(mmontan): add documentation for the module. +This module defines an opinionated setup of GKE +cluster. We outline project configurations, cluster settings, and basic K8s +objects that permit a safer-than-default configuration. -[^]: (autogen_docs_start) +## Module Usage -[^]: (autogen_docs_end) +The module fixes a set of parameters to values suggested in the +[GKE harderning guide](https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster), +the CIS framework, and other best practices. + +The motivation for each setting, and its relation to harderning guides or other recommendations +is outline in `main.tf` as comments over individual settings. When security-relevant settings +are available for configuration, recommendations on their settings are documented in the `variables.tf` file. + +## Project Setup and Cloud IAM policy for GKE + +### Applications and Clusters + +- Different applications that access data with different sensitivity and do + not need to communicate with each other (e.g., dev and prod instances of the + same application) should be placed in different clusters. + + - This approach will limit the blast radius of errors. An security problem + in dev shouldn't impact production data. + +- If applications need to communicate (e.g., a frontend system calling + a backend), we suggest placing the two applications in the same cluster, in + different namespaces. + + - Placing them in the same cluster will provide fast network + communication, and the different namespaces will be configured to + provide some administrative isolation. Istio will be used to encrypt and + control communication between applications. + +- We suggest to store user or business data persistently in managed storage + services that are inventories and controlled by centralized teams. + (e.g., GCP storage services within a GCP organization). + + - Storing user or business data securely requires satisfying a large set of + requirements, such as data inventory, which might be harder to satisfy at + scale when data is stored opaquely within a cluster. Services like Cloud + Asset Inventory provide centralized teams ability to enumerate data stores. + +### Project Setup + +We suggest a GKE setup composed of multiple projects to separate responsabilities +between cluster operators, which need to administer the cluster; and product +developers, which mostly just want to deploy and debug applications. + +- *Cluster Projects (`project_id`):* GKE clusters storing sensitive data should have their + own projects, so that they can be administered independently (e.g., dev cluster; + production clusters; staging clusters should go in different projects.) + +- *A shared GCR project (`registry_project_id`):* all clusters can share the same GCR project. + + - Easier to share images between environments. The same image could be + progressively rolled-out in dev, staging, and then production. + - Easier to manage service account permissions: GCR requires authorizing + service accounts to access certain buckets, which are created only after + images are published. When the only service run by the project is GCR, + we can safely give project-wide read permissions to all buckets. + +- (optional) *Data Projects:* When the same cluster is shared by different + applications managed by different teams, we suggest separating the data for + each application by placing storage resources for each team in different + projects (e.g., a Spanner instance for application A in one project, GCS + bucket for application B in a different project). + + - This permits to control administrative access to the data more tightly, + as Cloud IAM policies for accessing the data can be managed by each + application team, rather than the team managing the cluster + infrastructure. + +Exception to such a setup: + +- When not using Shared VPCs, resources that require direct network connectivity + (e.g., a Cloud SQL instance), need to be placed in the same VPC (hence, project) + as the clusters from which connections are made. + +### Google Service Accounts + +We use GKE Workload Identity (BETA) to associate a GCP identity to each workload, +and limit the permissions associated with the cluster nodes. + +The Safer Cluster setup relies on several service accounts: + +- The module generates a service account to run nodes. Such a service account + has only permissions of sending logging data, metrics, and downloading containers + from the given GCR project. The following settings in the module will create + a service account with the above properties: + +``` +create_service_account = true +registry_project_id = +grant_registry_access = true +``` + +- A service account *for each application* running on the cluster (e.g., + `myproduct-sa@myproduct-prod.iam.gserviceaccount.com`). These service + accounts should be associated to the permissions required for running the + application, such as access to databases. + +``` +- email: myproduct + displayName: Google Service Account for containers running in the myproduct k8s namespace + policy: + # GKE workload identity authorization. This authorizes the Kubernetes Service Account + # myproduct/default from this project's identity namespace to impersonate the service account. + # https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + bindings: + - members: + - serviceAccount:product-prod.svc.id.goog[myproduct/default] + role: roles/iam.workloadIdentityUser +``` + +We suggest running different applications in different namespaces within the cluster. Each namespace +should be assigned to its own GCP service account to better define the Cloud IAM permissions required +for the application. + +If you are using more than 2 projects in your setup, you can consider creting +the service account in a different project to keep application and +infrastructure separate. For example, service accounts could be created in each team's project, +while the cluster runs in a centrally controlled project. + +
+*Why?* + +Separating the permissions associated with the infrastructure GKE nodes and the +application provides a simpler way to scale up the cluster: multiple applications +could be run in the same cluster, and each of them can run with tailored permissions +that limit the impact of compromises. + +Such a separation of identities is enabled by a GKE feature called Workload +Identity. The feature provides additional advantages such as a better protection +of the node's metadata server against attackers. + +
+ +### Cloud IAM Permissions for the GKE Cluster + +We suggest to mainly rely on Kubernetes RBAC to manage access control, and use +Cloud IAM to give users only the ability of configuring `kubectl` credentials. + +Engineers operating applications on the cluster should only be assigned the +Cloud IAM permission `roles/container.clusterViewer`. This role allows them to +obtain credentials for the cluster, but provides no further access to the +cluster objects. All cluster objects are protected by RBAC configurations, +defined below. + +
+*Why?* + +Both Cloud IAM and RBAC can be used to control access to GKE clusters. Those two +systems are combined as a "OR": an action is authorized if the necessary +permissions are provided by either RBAC _OR_ Cloud IAM + +However, Cloud IAM permissions are defined for a project: user get assigned the +same permissions over all clusters and all namespaces within each cluster. Such +a setup makes it hard to separate responsibilities between teams in charge of +managing clusters, and teams in charge of products. + +By relying on RBAC instead of Cloud IAM, we have a finer-grained control of the +permissions provided to engineers, and permits to restrict permissions to only +certain namespaces. + +
+ +You can add the following binding to the `myproduct-prod` project. + +``` +- members: + role: roles/container.clusterViewer` + - group: + - group: +``` + +The permissions won't allow engineers to SSH into nodes as part of the regular +development workflow. Such permissions should be granted only to the cluster +team, and used only in case of emergency. + +While RBAC permissions should be sufficient for most cases, we also suggest to +create an emergency superuser role that can be used, given a proper +justification, for resolving cases where regular permissions are insufficient. +For simplicity, we suggest using `roles/container.admin` and +`roles/compute.admin`, until more narrow roles can be defined given your usage. + +``` +- members: + role: roles/container.admin + - group: +- members: + role: roles/compute.admin + - group: +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | string | `""` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index 920664593b..c2cdd7bca0 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -1,10 +1,10 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From 5a194719faa144ad0a7ee578663d336358f5073c Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Wed, 13 Nov 2019 14:03:23 -0600 Subject: [PATCH 08/15] address comments --- examples/safer_cluster/README.md | 15 ++++--- examples/safer_cluster/main.tf | 7 ++-- examples/safer_cluster/network.tf | 48 ++++++++++++++++++++++ examples/safer_cluster/outputs.tf | 10 +++++ examples/safer_cluster/variables.tf | 19 ++++++++- modules/safer-cluster/README.md | 9 ++-- modules/safer-cluster/main.tf | 17 +++----- modules/safer-cluster/variables.tf | 7 +++- test/ci/safer-cluster.yml | 18 -------- test/fixtures/safer_cluster/example.tf | 5 --- test/fixtures/safer_cluster/network.tf | 48 ---------------------- test/fixtures/safer_cluster/outputs.tf | 57 +++++++++++++++++++++++++- 12 files changed, 158 insertions(+), 102 deletions(-) create mode 100644 examples/safer_cluster/network.tf delete mode 100644 test/ci/safer-cluster.yml delete mode 100644 test/fixtures/safer_cluster/network.tf mode change 120000 => 100644 test/fixtures/safer_cluster/outputs.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 93a7d4a1a7..ba26e59314 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -10,14 +10,17 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | +| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | +| master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | n/a | yes | +| network | The VPC network to host the cluster in | string | `"gke-network"` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | `"us-central1"` | no | +| subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | +| subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | ## Outputs @@ -32,9 +35,11 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | +| network\_name | The name of the VPC being created | | project\_id | | | region | | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 102d539502..e865c9c2f9 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -20,7 +20,6 @@ locals { provider "google-beta" { version = "~> 2.18.0" - region = var.region } data "google_compute_subnetwork" "subnetwork" { @@ -35,8 +34,8 @@ module "gke" { name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" regional = true region = var.region - network = var.network - subnetwork = var.subnetwork + network = module.gcp-network.network_name + subnetwork = module.gcp-network.subnets_names[0] ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services compute_engine_service_account = var.compute_engine_service_account @@ -45,7 +44,7 @@ module "gke" { { cidr_blocks = [ { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + cidr_block = var.master_auth_subnetwork_cidr display_name = "VPC" }, ] diff --git a/examples/safer_cluster/network.tf b/examples/safer_cluster/network.tf new file mode 100644 index 0000000000..9bc38a9a3d --- /dev/null +++ b/examples/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 1.4.0" + project_id = var.project_id + network_name = var.network + + subnets = [ + { + subnet_name = var.subnetwork + subnet_ip = var.subnetwork_cidr + subnet_region = var.region + }, + { + subnet_name = var.master_auth_subnetwork + subnet_ip = var.master_auth_subnetwork_cidr + subnet_region = var.region + }, + ] + + secondary_ranges = { + "${var.subnetwork}" = [ + { + range_name = var.ip_range_pods + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = var.ip_range_services + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 2ff9f4734b..8a9914cdf4 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -36,3 +36,13 @@ output "service_account" { value = module.gke.service_account } +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index deb08f59e9..fc13c0a337 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -25,24 +25,39 @@ variable "cluster_name_suffix" { variable "region" { description = "The region to host the cluster in" + default = "us-central1" } variable "network" { description = "The VPC network to host the cluster in" + default = "gke-network" } variable "subnetwork" { description = "The subnetwork to host the cluster in" + default = "gke-subnet" +} +variable "subnetwork_cidr" { + description = "The cidr block for the subnetwork to host the cluster in" + default = "10.0.0.0/17" +} +variable "master_auth_subnetwork_cidr" { + description = "The cidr block for the subnetwork that has access to cluster master" + default = "10.60.0.0/17" +} +variable "master_auth_subnetwork" { + description = "The subnetwork that has access to cluster master" + default = "master-auth-subnet" } - variable "ip_range_pods" { description = "The secondary ip range to use for pods" + default = "ip-range-pods" } variable "ip_range_services" { description = "The secondary ip range to use for pods" + default = "ip-range-scv" } - variable "istio" { description = "Boolean to enable / disable Istio" default = true diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ad773982be..c4e3fcd41d 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,7 +1,5 @@ # Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -[TOC] - This module defines an opinionated setup of GKE cluster. We outline project configurations, cluster settings, and basic K8s objects that permit a safer-than-default configuration. @@ -37,7 +35,7 @@ are available for configuration, recommendations on their settings are documente control communication between applications. - We suggest to store user or business data persistently in managed storage - services that are inventories and controlled by centralized teams. + services that are inventoried and controlled by centralized teams. (e.g., GCP storage services within a GCP organization). - Storing user or business data securely requires satisfying a large set of @@ -47,7 +45,7 @@ are available for configuration, recommendations on their settings are documente ### Project Setup -We suggest a GKE setup composed of multiple projects to separate responsabilities +We suggest a GKE setup composed of multiple projects to separate responsibilities between cluster operators, which need to administer the cluster; and product developers, which mostly just want to deploy and debug applications. @@ -121,7 +119,7 @@ We suggest running different applications in different namespaces within the clu should be assigned to its own GCP service account to better define the Cloud IAM permissions required for the application. -If you are using more than 2 projects in your setup, you can consider creting +If you are using more than 2 projects in your setup, you can consider creating the service account in a different project to keep application and infrastructure separate. For example, service accounts could be created in each team's project, while the cluster runs in a centrally controlled project. @@ -239,6 +237,7 @@ For simplicity, we suggest using `roles/container.admin` and | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (required) | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 9069e22559..84c48b4a62 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -71,10 +71,6 @@ module "gke" { node_pools_taints = var.node_pools_taints node_pools_tags = var.node_pools_tags - // TODO(mmontan): we generally considered applying - // just the cloud-platoform scope and use Cloud IAM - // If we have Workload Identity, are there advantages - // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes stub_domains = var.stub_domains @@ -92,13 +88,10 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = var.compute_engine_service_account == "" ? false : true + create_service_account = var.compute_engine_service_account == "" ? true : false service_account = var.compute_engine_service_account - - // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, - // so that we can give GCS permissions to the service account on a project - // that hosts only container-images and not data. - grant_registry_access = true + registry_project_id = var.registry_project_id + grant_registry_access = true // Basic Auth disabled basic_auth_username = "" @@ -133,7 +126,7 @@ module "gke" { enable_binary_authorization = true // Define PodSecurityPolicies for differnet applications. - // TODO(mmontan): link to a couple of policies. + // Example: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#example pod_security_policy_config = [{ "enabled" = true }] @@ -144,7 +137,7 @@ module "gke" { // Sandbox can also provide increased protection in other cases, at some performance cost. sandbox_enabled = var.sandbox_enabled - // TODO(mmontan): investigate whether this should be a recommended setting + // Intranode Visibility enables you to capture flow logs for traffic between pods and create FW rules that apply to traffic between pods. enable_intranode_visibility = var.enable_intranode_visibility enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 71b4ba88ef..1b9d2d02d8 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -208,8 +208,11 @@ variable "grant_registry_access" { default = false } -// TODO(mmontan): allow specifying which project to use -// for reading images. +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} variable "service_account" { type = string diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml deleted file mode 100644 index ef3c896b29..0000000000 --- a/test/ci/safer-cluster.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - -platform: linux - -inputs: -- name: pull-request - path: terraform-google-kubernetes-engine - -run: - path: make - args: ['test_integration'] - dir: terraform-google-kubernetes-engine - -params: - SUITE: "safer-cluster-local" - COMPUTE_ENGINE_SERVICE_ACCOUNT: "" - REGION: "us-east4" - ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index c9d659ba67..a130769b42 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -18,11 +18,6 @@ module "example" { source = "../../../examples/safer_cluster" project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf deleted file mode 100644 index e1292eae3b..0000000000 --- a/test/fixtures/safer_cluster/network.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} - -provider "google" { - project = var.project_id -} - -resource "google_compute_network" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = var.region - network = google_compute_network.main.self_link - - secondary_ip_range { - range_name = "cft-gke-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "cft-gke-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} - diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf deleted file mode 120000 index 726bdc722f..0000000000 --- a/test/fixtures/safer_cluster/outputs.tf +++ /dev/null @@ -1 +0,0 @@ -../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 100644 index 0000000000..86b0648afd --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} +output "location" { + value = module.example.location +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} From fb3861210597273bfbdcf3732391f208d185b6bb Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Wed, 13 Nov 2019 15:32:05 -0600 Subject: [PATCH 09/15] fix lint --- examples/safer_cluster/README.md | 17 ----------------- examples/safer_cluster/outputs.tf | 4 ---- 2 files changed, 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index db76f2edb8..ba26e59314 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -10,7 +10,6 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -<<<<<<< HEAD | ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | | ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | | istio | Boolean to enable / disable Istio | string | `"true"` | no | @@ -22,16 +21,6 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | region | The region to host the cluster in | string | `"us-central1"` | no | | subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | | subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | -======= -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | -| istio | Boolean to enable / disable Istio | string | `"true"` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | n/a | yes | -| project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 ## Outputs @@ -46,17 +35,11 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | -<<<<<<< HEAD | network\_name | The name of the VPC being created | | project\_id | | | region | | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | -======= -| project\_id | | -| region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 6b42b830c7..eee2f4fbf2 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -36,7 +36,6 @@ output "service_account" { value = module.gke.service_account } -<<<<<<< HEAD output "network_name" { description = "The name of the VPC being created" value = module.gcp-network.network_name @@ -46,6 +45,3 @@ output "subnet_names" { description = "The names of the subnet being created" value = module.gcp-network.subnets_names } - -======= ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 From 1a5eb6354dc0281e7cf4c1a0f4e2159afed08e48 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 18 Nov 2019 08:19:13 -0600 Subject: [PATCH 10/15] add types, fix desc --- examples/safer_cluster/README.md | 4 +-- examples/safer_cluster/main.tf | 8 ++--- examples/safer_cluster/variables.tf | 14 ++++++++ modules/safer-cluster/README.md | 12 +++---- modules/safer-cluster/main.tf | 7 ++-- modules/safer-cluster/variables.tf | 12 +++---- test/fixtures/safer_cluster/network.tf | 48 -------------------------- 7 files changed, 33 insertions(+), 72 deletions(-) delete mode 100644 test/fixtures/safer_cluster/network.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index ba26e59314..845b263d13 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -7,12 +7,12 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cloudrun | Boolean to enable / disable CloudRun | bool | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | | ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | -| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| istio | Boolean to enable / disable Istio | bool | `"true"` | no | | master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | | master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index e865c9c2f9..bcae52aac9 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -18,14 +18,12 @@ locals { cluster_type = "safer-cluster" } -provider "google-beta" { +provider "google" { version = "~> 2.18.0" } -data "google_compute_subnetwork" "subnetwork" { - name = var.subnetwork - project = var.project_id - region = var.region +provider "google-beta" { + version = "~> 2.18.0" } module "gke" { diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index fc13c0a337..6b726747ca 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -15,64 +15,78 @@ */ variable "project_id" { + type = string description = "The project ID to host the cluster in" } variable "cluster_name_suffix" { + type = string description = "A suffix to append to the default cluster name" default = "" } variable "region" { + type = string description = "The region to host the cluster in" default = "us-central1" } variable "network" { + type = string description = "The VPC network to host the cluster in" default = "gke-network" } variable "subnetwork" { + type = string description = "The subnetwork to host the cluster in" default = "gke-subnet" } variable "subnetwork_cidr" { + type = string description = "The cidr block for the subnetwork to host the cluster in" default = "10.0.0.0/17" } variable "master_auth_subnetwork_cidr" { + type = string description = "The cidr block for the subnetwork that has access to cluster master" default = "10.60.0.0/17" } variable "master_auth_subnetwork" { + type = string description = "The subnetwork that has access to cluster master" default = "master-auth-subnet" } variable "ip_range_pods" { + type = string description = "The secondary ip range to use for pods" default = "ip-range-pods" } variable "ip_range_services" { + type = string description = "The secondary ip range to use for pods" default = "ip-range-scv" } variable "istio" { + type = bool description = "Boolean to enable / disable Istio" default = true } variable "cloudrun" { + type = bool description = "Boolean to enable / disable CloudRun" default = true } variable "master_ipv4_cidr_block" { + type = string description = "The IP range in CIDR notation to use for the hosted master network" default = "172.16.0.0/28" } variable "compute_engine_service_account" { + type = string description = "Service account to associate to the nodes in the cluster" } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index c4e3fcd41d..4b51d4d5e3 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -224,8 +224,8 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| name | The name of the cluster | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | @@ -234,17 +234,17 @@ For simplicity, we suggest using `roles/container.admin` and | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | +| zones | The zones to host the cluster in | list(string) | `` | no | ## Outputs diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 84c48b4a62..76c40b3ebe 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -62,11 +62,8 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - node_pools = var.node_pools - node_pools_labels = var.node_pools_labels - - // TODO(mmontan): check whether we need to restrict these - // settings. + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels node_pools_metadata = var.node_pools_metadata node_pools_taints = var.node_pools_taints node_pools_tags = var.node_pools_tags diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 1b9d2d02d8..fe47be7be6 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -18,12 +18,12 @@ variable "project_id" { type = string - description = "The project ID to host the cluster in (required)" + description = "The project ID to host the cluster in" } variable "name" { type = string - description = "The name of the cluster (required)" + description = "The name of the cluster" } variable "description" { @@ -40,18 +40,18 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in" } variable "zones" { type = list(string) - description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + description = "The zones to host the cluster in" default = [] } variable "network" { type = string - description = "The VPC network to host the cluster in (required)" + description = "The VPC network to host the cluster in" } variable "network_project_id" { @@ -62,7 +62,7 @@ variable "network_project_id" { variable "subnetwork" { type = string - description = "The subnetwork to host the cluster in (required)" + description = "The subnetwork to host the cluster in" } variable "kubernetes_version" { diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf deleted file mode 100644 index e1292eae3b..0000000000 --- a/test/fixtures/safer_cluster/network.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} - -provider "google" { - project = var.project_id -} - -resource "google_compute_network" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = var.region - network = google_compute_network.main.self_link - - secondary_ip_range { - range_name = "cft-gke-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "cft-gke-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} - From 1f2edb63d59bc257809b92fad6b25bc4ec9880ae Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 22 Nov 2019 08:32:15 -0600 Subject: [PATCH 11/15] remove kubernetes_dashboard_enabled --- modules/safer-cluster/README.md | 1 - modules/safer-cluster/outputs.tf | 5 ----- 2 files changed, 6 deletions(-) diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 4b51d4d5e3..2d99d26f36 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -254,7 +254,6 @@ For simplicity, we suggest using `roles/container.admin` and | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf index bb4fb79667..1c688e27d5 100644 --- a/modules/safer-cluster/outputs.tf +++ b/modules/safer-cluster/outputs.tf @@ -102,11 +102,6 @@ output "horizontal_pod_autoscaling_enabled" { value = module.gke.horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = module.gke.kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = module.gke.node_pools_names From 509f6badc4abe5c11b02a300352b47ca3fed4b79 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 22 Nov 2019 09:23:10 -0600 Subject: [PATCH 12/15] fix int test --- build/int.cloudbuild.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index af0560641f..45e23b6193 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -64,6 +64,26 @@ steps: - verify shared-vpc-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy shared-vpc-local'] +- id: create safer-cluster-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create safer-cluster-local'] +- id: converge safer-cluster-local + waitFor: + - create safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge safer-cluster-local'] +- id: verify safer-cluster-local + waitFor: + - converge safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify safer-cluster-local'] +- id: destroy safer-cluster-local + waitFor: + - verify safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy safer-cluster-local'] - id: create simple-regional-local waitFor: - prepare From 932bd0e464370a0b772291bad1e45dcd6aa051d2 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Sat, 23 Nov 2019 12:30:33 -0600 Subject: [PATCH 13/15] hardcode vars --- examples/safer_cluster/README.md | 20 +-------- examples/safer_cluster/main.tf | 27 ++++++++---- examples/safer_cluster/network.tf | 16 +++---- examples/safer_cluster/outputs.tf | 24 ++++++++++ examples/safer_cluster/test_outputs.tf | 1 - examples/safer_cluster/variables.tf | 61 -------------------------- 6 files changed, 53 insertions(+), 96 deletions(-) delete mode 120000 examples/safer_cluster/test_outputs.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 845b263d13..2e925ac85b 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -7,20 +7,9 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cloudrun | Boolean to enable / disable CloudRun | bool | `"true"` | no | -| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | -| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | -| istio | Boolean to enable / disable Istio | bool | `"true"` | no | -| master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | -| master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | `"gke-network"` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | `"us-central1"` | no | -| subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | -| subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | ## Outputs @@ -29,18 +18,13 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | ca\_certificate | The cluster ca certificate (base64 encoded) | | client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| ip\_range\_pods | The secondary IP range used for pods | -| ip\_range\_services | The secondary IP range used for services | | kubernetes\_endpoint | The cluster endpoint | | location | | -| master\_kubernetes\_version | The master Kubernetes version | -| network | | +| master\_kubernetes\_version | Kubernetes version of the master | | network\_name | The name of the VPC being created | -| project\_id | | -| region | | +| region | The region in which the cluster resides | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | -| subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index bcae52aac9..0aeca18009 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -14,8 +14,19 @@ * limitations under the License. */ +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + locals { - cluster_type = "safer-cluster" + cluster_type = "safer-cluster" + network_name = "safer-cluster-network-${random_string.suffix.result}" + subnet_name = "safer-cluster-subnet-${random_string.suffix.result}" + master_auth_subnetwork = "safer-cluster-master-subnet-${random_string.suffix.result}" + pods_range_name = "ip-range-pods-${random_string.suffix.result}" + svc_range_name = "ip-range-svc-${random_string.suffix.result}" } provider "google" { @@ -29,27 +40,27 @@ provider "google-beta" { module "gke" { source = "../../modules/safer-cluster/" project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + name = "${local.cluster_type}-cluster-${random_string.suffix.result}" regional = true region = var.region network = module.gcp-network.network_name subnetwork = module.gcp-network.subnets_names[0] - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name compute_engine_service_account = var.compute_engine_service_account - master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_ipv4_cidr_block = "172.16.0.0/28" master_authorized_networks_config = [ { cidr_blocks = [ { - cidr_block = var.master_auth_subnetwork_cidr + cidr_block = "10.60.0.0/17" display_name = "VPC" }, ] }, ] - istio = var.istio - cloudrun = var.cloudrun + istio = true + cloudrun = true } data "google_client_config" "default" { diff --git a/examples/safer_cluster/network.tf b/examples/safer_cluster/network.tf index 9bc38a9a3d..7ce9cf3946 100644 --- a/examples/safer_cluster/network.tf +++ b/examples/safer_cluster/network.tf @@ -18,29 +18,29 @@ module "gcp-network" { source = "terraform-google-modules/network/google" version = "~> 1.4.0" project_id = var.project_id - network_name = var.network + network_name = local.network_name subnets = [ { - subnet_name = var.subnetwork - subnet_ip = var.subnetwork_cidr + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" subnet_region = var.region }, { - subnet_name = var.master_auth_subnetwork - subnet_ip = var.master_auth_subnetwork_cidr + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" subnet_region = var.region }, ] secondary_ranges = { - "${var.subnetwork}" = [ + "${local.subnet_name}" = [ { - range_name = var.ip_range_pods + range_name = local.pods_range_name ip_cidr_range = "192.168.0.0/18" }, { - range_name = var.ip_range_services + range_name = local.svc_range_name ip_cidr_range = "192.168.64.0/18" }, ] diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index eee2f4fbf2..72e09c0467 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -20,6 +20,20 @@ output "kubernetes_endpoint" { value = module.gke.endpoint } +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + output "client_token" { description = "The bearer token for auth" sensitive = true @@ -45,3 +59,13 @@ output "subnet_names" { description = "The names of the subnet being created" value = module.gcp-network.subnets_names } + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf deleted file mode 120000 index 17b34213ba..0000000000 --- a/examples/safer_cluster/test_outputs.tf +++ /dev/null @@ -1 +0,0 @@ -../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 6b726747ca..dc3c20889a 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -19,73 +19,12 @@ variable "project_id" { description = "The project ID to host the cluster in" } -variable "cluster_name_suffix" { - type = string - description = "A suffix to append to the default cluster name" - default = "" -} - variable "region" { type = string description = "The region to host the cluster in" default = "us-central1" } -variable "network" { - type = string - description = "The VPC network to host the cluster in" - default = "gke-network" -} - -variable "subnetwork" { - type = string - description = "The subnetwork to host the cluster in" - default = "gke-subnet" -} -variable "subnetwork_cidr" { - type = string - description = "The cidr block for the subnetwork to host the cluster in" - default = "10.0.0.0/17" -} -variable "master_auth_subnetwork_cidr" { - type = string - description = "The cidr block for the subnetwork that has access to cluster master" - default = "10.60.0.0/17" -} -variable "master_auth_subnetwork" { - type = string - description = "The subnetwork that has access to cluster master" - default = "master-auth-subnet" -} -variable "ip_range_pods" { - type = string - description = "The secondary ip range to use for pods" - default = "ip-range-pods" -} - -variable "ip_range_services" { - type = string - description = "The secondary ip range to use for pods" - default = "ip-range-scv" -} -variable "istio" { - type = bool - description = "Boolean to enable / disable Istio" - default = true -} - -variable "cloudrun" { - type = bool - description = "Boolean to enable / disable CloudRun" - default = true -} - -variable "master_ipv4_cidr_block" { - type = string - description = "The IP range in CIDR notation to use for the hosted master network" - default = "172.16.0.0/28" -} - variable "compute_engine_service_account" { type = string description = "Service account to associate to the nodes in the cluster" From b06c42541ca2fd830332635f58d3bb5818db77e3 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 25 Nov 2019 23:10:50 -0600 Subject: [PATCH 14/15] update int test to split between two projects --- examples/safer_cluster/README.md | 1 + examples/safer_cluster/outputs.tf | 5 ++++ test/fixtures/deploy_service/example.tf | 4 +-- test/fixtures/deploy_service/network.tf | 2 +- test/fixtures/disable_client_cert/example.tf | 6 ++-- test/fixtures/disable_client_cert/network.tf | 2 +- test/fixtures/node_pool/example.tf | 4 +-- test/fixtures/node_pool/network.tf | 2 +- .../node_pool_update_variant/example.tf | 4 +-- .../node_pool_update_variant/network.tf | 2 +- .../private_zonal_with_networking/example.tf | 2 +- .../private_zonal_with_networking/outputs.tf | 2 +- .../variables.tf | 3 +- test/fixtures/safer_cluster/example.tf | 4 +-- test/fixtures/safer_cluster/outputs.tf | 2 +- test/fixtures/sandbox_enabled/example.tf | 4 +-- test/fixtures/sandbox_enabled/network.tf | 2 +- test/fixtures/shared/outputs.tf | 2 +- test/fixtures/shared/variables.tf | 10 ++++--- test/fixtures/shared_vpc/example.tf | 6 ++-- test/fixtures/shared_vpc/network.tf | 2 +- test/fixtures/simple_regional/example.tf | 4 +-- test/fixtures/simple_regional/network.tf | 2 +- .../simple_regional_private/example.tf | 4 +-- .../simple_regional_private/network.tf | 4 +-- .../example.tf | 2 +- .../outputs.tf | 2 +- .../variables.tf | 5 ++-- test/fixtures/simple_zonal/example.tf | 2 +- test/fixtures/simple_zonal/network.tf | 2 +- test/fixtures/simple_zonal_private/example.tf | 4 +-- test/fixtures/simple_zonal_private/network.tf | 4 +-- test/fixtures/stub_domains/example.tf | 4 +-- test/fixtures/stub_domains/network.tf | 2 +- test/fixtures/stub_domains_private/main.tf | 8 ++--- .../example.tf | 4 +-- .../network.tf | 2 +- test/fixtures/upstream_nameservers/example.tf | 4 +-- test/fixtures/upstream_nameservers/network.tf | 2 +- .../workload_metadata_config/example.tf | 2 +- .../workload_metadata_config/network.tf | 6 ++-- test/setup/iam.tf | 28 ++++++++++++----- test/setup/main.tf | 30 ++++++++++++++++++- test/setup/outputs.tf | 8 ++--- 44 files changed, 129 insertions(+), 77 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 2e925ac85b..e400641ab9 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -22,6 +22,7 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | Kubernetes version of the master | | network\_name | The name of the VPC being created | +| project\_id | The project ID the cluster is in | | region | The region in which the cluster resides | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 72e09c0467..d2f9706f02 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -69,3 +69,8 @@ output "zones" { description = "List of zones in which the cluster resides" value = module.gke.zones } + +output "project_id" { + description = "The project ID the cluster is in" + value = var.project_id +} diff --git a/test/fixtures/deploy_service/example.tf b/test/fixtures/deploy_service/example.tf index 421c551489..60904163a8 100644 --- a/test/fixtures/deploy_service/example.tf +++ b/test/fixtures/deploy_service/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/deploy_service" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/deploy_service/network.tf b/test/fixtures/deploy_service/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/deploy_service/network.tf +++ b/test/fixtures/deploy_service/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/disable_client_cert/example.tf b/test/fixtures/disable_client_cert/example.tf index 23ea6da936..5bacef9fb9 100644 --- a/test/fixtures/disable_client_cert/example.tf +++ b/test/fixtures/disable_client_cert/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/disable_client_cert" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name - network_project_id = var.project_id + network_project_id = var.project_ids[0] subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/disable_client_cert/network.tf b/test/fixtures/disable_client_cert/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/disable_client_cert/network.tf +++ b/test/fixtures/disable_client_cert/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 28d51f2357..82dd01035c 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/node_pool" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,6 +25,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/node_pool/network.tf b/test/fixtures/node_pool/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/node_pool/network.tf +++ b/test/fixtures/node_pool/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/node_pool_update_variant/example.tf b/test/fixtures/node_pool_update_variant/example.tf index c3a21df3d5..b7f9c8c390 100644 --- a/test/fixtures/node_pool_update_variant/example.tf +++ b/test/fixtures/node_pool_update_variant/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/node_pool_update_variant" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,5 +25,5 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/node_pool_update_variant/network.tf b/test/fixtures/node_pool_update_variant/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/node_pool_update_variant/network.tf +++ b/test/fixtures/node_pool_update_variant/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/private_zonal_with_networking/example.tf b/test/fixtures/private_zonal_with_networking/example.tf index 23612f1610..87bf06ace4 100644 --- a/test/fixtures/private_zonal_with_networking/example.tf +++ b/test/fixtures/private_zonal_with_networking/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/private_zonal_with_networking" - project_id = var.project_id + project_id = var.project_ids[0] region = var.region zones = var.zones } diff --git a/test/fixtures/private_zonal_with_networking/outputs.tf b/test/fixtures/private_zonal_with_networking/outputs.tf index 08f9a8a2e8..4f9edf51a0 100644 --- a/test/fixtures/private_zonal_with_networking/outputs.tf +++ b/test/fixtures/private_zonal_with_networking/outputs.tf @@ -16,7 +16,7 @@ output "project_id" { - value = var.project_id + value = module.example.project_id } output "location" { diff --git a/test/fixtures/private_zonal_with_networking/variables.tf b/test/fixtures/private_zonal_with_networking/variables.tf index d610f1e807..5719be3c18 100644 --- a/test/fixtures/private_zonal_with_networking/variables.tf +++ b/test/fixtures/private_zonal_with_networking/variables.tf @@ -14,7 +14,8 @@ * limitations under the License. */ -variable "project_id" { +variable "project_ids" { + type = list(string) description = "The project ID to host the cluster in" } diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index a130769b42..8cd7983c27 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id + project_id = var.project_ids[0] region = var.region - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf index 86b0648afd..6613bcb29a 100644 --- a/test/fixtures/safer_cluster/outputs.tf +++ b/test/fixtures/safer_cluster/outputs.tf @@ -15,7 +15,7 @@ */ output "project_id" { - value = var.project_id + value = module.example.project_id } output "region" { diff --git a/test/fixtures/sandbox_enabled/example.tf b/test/fixtures/sandbox_enabled/example.tf index 05b7edfd9e..920c5e179f 100644 --- a/test/fixtures/sandbox_enabled/example.tf +++ b/test/fixtures/sandbox_enabled/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/simple_regional_beta" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] istio = false cloudrun = false node_metadata = "UNSPECIFIED" diff --git a/test/fixtures/sandbox_enabled/network.tf b/test/fixtures/sandbox_enabled/network.tf index 5d34d43748..f170a88fa4 100644 --- a/test/fixtures/sandbox_enabled/network.tf +++ b/test/fixtures/sandbox_enabled/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 71b4b250de..dfc0d02396 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -15,7 +15,7 @@ */ output "project_id" { - value = var.project_id + value = module.example.project_id } output "region" { diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index 9760d65a94..56e5b63441 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -14,8 +14,9 @@ * limitations under the License. */ -variable "project_id" { - description = "The GCP project to use for integration tests" +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" } variable "region" { @@ -29,8 +30,9 @@ variable "zones" { default = ["us-central1-a", "us-central1-b", "us-central1-c"] } -variable "compute_engine_service_account" { - description = "The email address of the service account to associate with the GKE cluster" +variable "compute_engine_service_accounts" { + type = list(string) + description = "The email addresses of the service account to associate with the GKE cluster" } variable "registry_project_id" { diff --git a/test/fixtures/shared_vpc/example.tf b/test/fixtures/shared_vpc/example.tf index 051d9155c9..a91d50fe85 100644 --- a/test/fixtures/shared_vpc/example.tf +++ b/test/fixtures/shared_vpc/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/shared_vpc" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name - network_project_id = var.project_id + network_project_id = var.project_ids[0] subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/shared_vpc/network.tf b/test/fixtures/shared_vpc/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/shared_vpc/network.tf +++ b/test/fixtures/shared_vpc/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index 7f8bb83637..2e08ae6628 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/simple_regional" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] skip_provisioners = true } diff --git a/test/fixtures/simple_regional/network.tf b/test/fixtures/simple_regional/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/simple_regional/network.tf +++ b/test/fixtures/simple_regional/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_regional_private/example.tf b/test/fixtures/simple_regional_private/example.tf index c06f080ac4..e7c71a3c64 100644 --- a/test/fixtures/simple_regional_private/example.tf +++ b/test/fixtures/simple_regional_private/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/simple_regional_private" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/simple_regional_private/network.tf b/test/fixtures/simple_regional_private/network.tf index f34f629069..8d643281e1 100644 --- a/test/fixtures/simple_regional_private/network.tf +++ b/test/fixtures/simple_regional_private/network.tf @@ -21,13 +21,13 @@ resource "random_string" "suffix" { } resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf index c7ae5af76c..d8d2172341 100644 --- a/test/fixtures/simple_regional_with_networking/example.tf +++ b/test/fixtures/simple_regional_with_networking/example.tf @@ -17,6 +17,6 @@ module "example" { source = "../../../examples/simple_regional_with_networking" - project_id = var.project_id + project_id = var.project_ids[1] region = var.region } diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf index 08f9a8a2e8..4f9edf51a0 100644 --- a/test/fixtures/simple_regional_with_networking/outputs.tf +++ b/test/fixtures/simple_regional_with_networking/outputs.tf @@ -16,7 +16,7 @@ output "project_id" { - value = var.project_id + value = module.example.project_id } output "location" { diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf index e9310a56c5..9bbb43040c 100644 --- a/test/fixtures/simple_regional_with_networking/variables.tf +++ b/test/fixtures/simple_regional_with_networking/variables.tf @@ -14,8 +14,9 @@ * limitations under the License. */ -variable "project_id" { - description = "The project ID to host the cluster in" +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" } variable "region" { diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf index 7d558e36f0..5498b28b00 100644 --- a/test/fixtures/simple_zonal/example.tf +++ b/test/fixtures/simple_zonal/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/simple_zonal_with_acm" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) diff --git a/test/fixtures/simple_zonal/network.tf b/test/fixtures/simple_zonal/network.tf index a4978c9ac3..e0bf46c2f2 100644 --- a/test/fixtures/simple_zonal/network.tf +++ b/test/fixtures/simple_zonal/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_zonal_private/example.tf b/test/fixtures/simple_zonal_private/example.tf index 0f8a4fc478..3ccce8ceab 100644 --- a/test/fixtures/simple_zonal_private/example.tf +++ b/test/fixtures/simple_zonal_private/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/simple_zonal_private" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,6 +25,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/simple_zonal_private/network.tf b/test/fixtures/simple_zonal_private/network.tf index 76d33f6bfc..fc7de87ab4 100644 --- a/test/fixtures/simple_zonal_private/network.tf +++ b/test/fixtures/simple_zonal_private/network.tf @@ -22,13 +22,13 @@ resource "random_string" "suffix" { resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/fixtures/stub_domains/example.tf b/test/fixtures/stub_domains/example.tf index 32cb31460e..df31547535 100644 --- a/test/fixtures/stub_domains/example.tf +++ b/test/fixtures/stub_domains/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/stub_domains" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/stub_domains/network.tf b/test/fixtures/stub_domains/network.tf index e1292eae3b..a24129ec4f 100644 --- a/test/fixtures/stub_domains/network.tf +++ b/test/fixtures/stub_domains/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/stub_domains_private/main.tf b/test/fixtures/stub_domains_private/main.tf index 101aac220a..a98f8d6ba6 100644 --- a/test/fixtures/stub_domains_private/main.tf +++ b/test/fixtures/stub_domains_private/main.tf @@ -24,7 +24,7 @@ resource "google_compute_network" "main" { name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false - project = var.project_id + project = var.project_ids[1] } resource "google_compute_subnetwork" "main" { @@ -32,7 +32,7 @@ resource "google_compute_subnetwork" "main" { name = "cft-gke-test-${random_string.suffix.result}" network = google_compute_network.main.self_link - project = var.project_id + project = var.project_ids[1] region = var.region secondary_ip_range { @@ -49,11 +49,11 @@ resource "google_compute_subnetwork" "main" { module "example" { source = "../../../examples/stub_domains_private" - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name network = google_compute_network.main.name - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region subnetwork = google_compute_subnetwork.main.name diff --git a/test/fixtures/stub_domains_upstream_nameservers/example.tf b/test/fixtures/stub_domains_upstream_nameservers/example.tf index a066994219..a6923c0f13 100644 --- a/test/fixtures/stub_domains_upstream_nameservers/example.tf +++ b/test/fixtures/stub_domains_upstream_nameservers/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/stub_domains_upstream_nameservers" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/stub_domains_upstream_nameservers/network.tf b/test/fixtures/stub_domains_upstream_nameservers/network.tf index ee202615ea..8ec5389ade 100644 --- a/test/fixtures/stub_domains_upstream_nameservers/network.tf +++ b/test/fixtures/stub_domains_upstream_nameservers/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/upstream_nameservers/example.tf b/test/fixtures/upstream_nameservers/example.tf index 72e6307fb1..81d60e8559 100644 --- a/test/fixtures/upstream_nameservers/example.tf +++ b/test/fixtures/upstream_nameservers/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/upstream_nameservers" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/upstream_nameservers/network.tf b/test/fixtures/upstream_nameservers/network.tf index ee202615ea..8ec5389ade 100644 --- a/test/fixtures/upstream_nameservers/network.tf +++ b/test/fixtures/upstream_nameservers/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/workload_metadata_config/example.tf b/test/fixtures/workload_metadata_config/example.tf index 3568cfa404..ea9519579d 100644 --- a/test/fixtures/workload_metadata_config/example.tf +++ b/test/fixtures/workload_metadata_config/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/workload_metadata_config" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) diff --git a/test/fixtures/workload_metadata_config/network.tf b/test/fixtures/workload_metadata_config/network.tf index f163eb7730..f8a3322a65 100644 --- a/test/fixtures/workload_metadata_config/network.tf +++ b/test/fixtures/workload_metadata_config/network.tf @@ -21,17 +21,17 @@ resource "random_string" "suffix" { } provider "google-beta" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = "false" } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 7ff4de74bc..b26c46568e 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -36,21 +36,35 @@ resource "random_id" "random_suffix" { } resource "google_service_account" "int_test" { - project = module.gke-project.project_id + project = module.gke-project-1.project_id account_id = "gke-int-test-${random_id.random_suffix.hex}" display_name = "gke-int-test" } -resource "google_service_account" "gke_sa" { - project = module.gke-project.project_id - account_id = "gke-sa-int-test-${random_id.random_suffix.hex}" - display_name = "gke-sa-int-test" +resource "google_service_account" "gke_sa_1" { + project = module.gke-project-1.project_id + account_id = "gke-sa-int-test-p1-${random_id.random_suffix.hex}" + display_name = "gke-sa-int-test-p1" } -resource "google_project_iam_member" "int_test" { +resource "google_service_account" "gke_sa_2" { + project = module.gke-project-2.project_id + account_id = "gke-sa-int-test-p2-${random_id.random_suffix.hex}" + display_name = "gke-sa-int-test-p2" +} + +resource "google_project_iam_member" "int_test_1" { + count = length(local.int_required_roles) + + project = module.gke-project-1.project_id + role = local.int_required_roles[count.index] + member = "serviceAccount:${google_service_account.int_test.email}" +} + +resource "google_project_iam_member" "int_test_2" { count = length(local.int_required_roles) - project = module.gke-project.project_id + project = module.gke-project-2.project_id role = local.int_required_roles[count.index] member = "serviceAccount:${google_service_account.int_test.email}" } diff --git a/test/setup/main.tf b/test/setup/main.tf index 70e10c46a3..961d1b47ac 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -14,7 +14,35 @@ * limitations under the License. */ -module "gke-project" { +module "gke-project-1" { + source = "terraform-google-modules/project-factory/google" + version = "~> 3.0" + + name = "ci-gke" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + auto_create_network = true + + activate_apis = [ + "bigquery-json.googleapis.com", + "cloudkms.googleapis.com", + "cloudresourcemanager.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "containerregistry.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "oslogin.googleapis.com", + "pubsub.googleapis.com", + "serviceusage.googleapis.com", + "storage-api.googleapis.com", + ] +} + +module "gke-project-2" { source = "terraform-google-modules/project-factory/google" version = "~> 3.0" diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 3e508ed1c7..b0f9877276 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -14,8 +14,8 @@ * limitations under the License. */ -output "project_id" { - value = module.gke-project.project_id +output "project_ids" { + value = [module.gke-project-1.project_id, module.gke-project-2.project_id] } output "sa_key" { @@ -23,6 +23,6 @@ output "sa_key" { sensitive = true } -output "compute_engine_service_account" { - value = google_service_account.gke_sa.email +output "compute_engine_service_accounts" { + value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email] } From 0be30c8fd6acd46684d830b236e1806c17ba3897 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Tue, 26 Nov 2019 00:05:16 -0600 Subject: [PATCH 15/15] fix outputs --- test/setup/make_source.sh | 12 ++++++------ test/setup/outputs.tf | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh index ad3f57165a..4af7cd63cd 100755 --- a/test/setup/make_source.sh +++ b/test/setup/make_source.sh @@ -16,15 +16,15 @@ echo "#!/usr/bin/env bash" > ../source.sh -project_id=$(terraform output project_id) -echo "export TF_VAR_project_id='$project_id'" >> ../source.sh +project_ids=$(terraform output project_ids) +echo "export TF_VAR_project_ids='$project_ids'" >> ../source.sh -# We use the same project for registry project in the tests. -echo "export TF_VAR_registry_project_id='$project_id'" >> ../source.sh +registry_project_id=$(terraform output registry_project_id) +echo "export TF_VAR_registry_project_id='$registry_project_id'" >> ../source.sh sa_json=$(terraform output sa_key) # shellcheck disable=SC2086 echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh -compute_engine_service_account=$(terraform output compute_engine_service_account) -echo "export TF_VAR_compute_engine_service_account='$compute_engine_service_account'" >> ../source.sh +compute_engine_service_accounts=$(terraform output compute_engine_service_accounts) +echo "export TF_VAR_compute_engine_service_accounts='$compute_engine_service_accounts'" >> ../source.sh diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index b0f9877276..ddebd75861 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -26,3 +26,7 @@ output "sa_key" { output "compute_engine_service_accounts" { value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email] } + +output "registry_project_id" { + value = module.gke-project-1.project_id +}