diff --git a/.gitignore b/.gitignore index 7fab33f949..899be3e749 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ test/integration/gcloud/config.sh test/integration/tmp credentials.json + +# File to populate env vars used by Docker test runs +.envrc diff --git a/CHANGELOG.md b/CHANGELOG.md index 714e2df341..92a7190bf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,44 +1,98 @@ # Change Log - All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](http://keepachangelog.com/) and this -project adheres to [Semantic Versioning](http://semver.org/). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Extending the adopted spec, each change should have a link to its corresponding pull request appended. ## [Unreleased] + +## [v1.0.0] - 2019-03-25 +### Added +* Allow creation of service accounts. [#80] +* Add support for private clusters via submodule. [#69] +* Add `remove_default_node_pool` set to `false` by default. Fixes [#15]. [#55] +* Allow arbitrary key-value pairs to be set on node pool metadata. [#52] +* Add `initial_node_count` parameter to node_pool block. [#60] +* Added `disable_legacy_metadata_endpoints` parameter. [#114] + ### Changed -* Add support for private clusters via submodule. #69 -* Set `horizontal_pod_autoscaling` to `true` by default. #42 -* Add `remove_default_node_pool` set to `false` by default #15 -* Allow arbitrary key-value pairs to be set on node pool metadata. #52 -* Add `initial_node_count` parameter to node_pool block. #60 +* Set `horizontal_pod_autoscaling` to `true` by default. Fixes [#42]. [#54] +* Update simple-zonal example GKE version to supported version. [#49] +* Drop explicit version from simple_zonal example. [#74] +* Remove explicit versions from test cases and examples. [#62] +* Set up submodule structure for public and private clusters. [#61] +* Update the google and google-beta providers to v2.2 [#106] + +### Fixed +* Zonal clusters can now accept a single zone. Fixes [#43]. [#50] +* Fix link to "configure a service account" [#73] +* Fix issue with regional cluster roll outs causing version skews [#108] +* Fix permanent metadata skew due to disable-legacy-endpoints keys [#114] ## [v0.4.0] - 2018-12-19 ### Added -* Added support for testing with kitchen-terraform. #33 -* Added support for preemptible nodes. #38 +* Added support for testing with kitchen-terraform. [#33] +* Added support for preemptible nodes. [#38] ### Changed -* Updated default version to `1.10.6`. #31 +* Updated default version to `1.10.6`. [#31] ### Fixed -* `region` argument on google_compute_subnetwork caused errors. #22 -* Added check to wait for GKE cluster to be `READY` before completing. #46 +* `region` argument on google_compute_subnetwork caused errors. [#22] +* Added check to wait for GKE cluster to be `READY` before completing. [#46] ## [v0.3.0] - 2018-10-10 ### Changed -* Updated network/subnetwork lookup to use data source. #16 -* Make zone configuration optional when creating a regional cluster. #19 +* Updated network/subnetwork lookup to use data source. [#16] +* Make zone configuration optional when creating a regional cluster. [#19] ## [v0.2.0] - 2018-09-26 ### Added -* Support for configuring master authorized networks. (#10) -* Support specifying monitoring and logging services. (#9) +* Support for configuring master authorized networks. [#10] +* Support specifying monitoring and logging services. [#9] -## [v0.1.0] - 2018-09-12 +## v0.1.0 - 2018-09-12 ### Added * Initial release of module. + +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v1.0.0...HEAD +[v1.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.4.0...v1.0.0 +[v0.4.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.4.0...v0.5.0 +[v0.4.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.3.0...v0.4.0 +[v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 +[v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 + +[#43]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/43 +[#42]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/42 +[#15]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/15 + +[#109]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/109 +[#108]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/108 +[#106]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/106 +[#80]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/80 +[#74]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/74 +[#73]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/73 +[#61]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/61 +[#69]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/69 +[#62]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/62 +[#60]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/60 +[#55]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/55 +[#54]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/54 +[#52]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/52 +[#50]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/50 +[#49]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/49 +[#46]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/46 +[#38]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/38 +[#33]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/33 +[#31]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/31 +[#22]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/22 +[#19]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/19 +[#16]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/16 +[#10]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/10 +[#9]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/9 diff --git a/Makefile b/Makefile index cd77f45345..4a78f1eb7a 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ docker_create: docker_build_kitchen_terraform -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ -v $(CURDIR):/cft/workdir \ ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen create" + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen create" .PHONY: docker_converge docker_converge: @@ -144,7 +144,7 @@ docker_converge: -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ -v $(CURDIR):/cft/workdir \ ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen converge && kitchen converge" + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen converge && kitchen converge" .PHONY: docker_verify docker_verify: @@ -158,7 +158,7 @@ docker_verify: -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ -v $(CURDIR):/cft/workdir \ ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen verify" + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen verify" .PHONY: docker_destroy docker_destroy: @@ -172,7 +172,7 @@ docker_destroy: -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ -v $(CURDIR):/cft/workdir \ ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen destroy" + /bin/bash -c "source test/ci_integration.sh && setup_environment && kitchen destroy" .PHONY: test_integration_docker test_integration_docker: diff --git a/README.md b/README.md index f76c1e6805..6fece407ce 100644 --- a/README.md +++ b/README.md @@ -89,67 +89,73 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure +## Upgrade to v1.0.0 -[^]: (autogen_docs_start) +Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. + +In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +[^]: (autogen_docs_start) ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| description | The description of the cluster | string | `` | no | -| horizontal_pod_autoscaling | Enable horizontal pod autoscaling addon | string | `true` | no | -| http_load_balancing | Enable httpload balancer addon | string | `true` | no | -| ip_masq_link_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `false` | no | -| ip_masq_resync_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `60s` | no | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no | -| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no | -| logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no | -| maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no | -| master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `monitoring.googleapis.com` | no | -| name | The name of the cluster (required) | string | - | yes | -| network | The VPC network to host the cluster in (required) | string | - | yes | -| network_policy | Enable network policy addon | string | `false` | no | -| network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | -| node_pools | List of maps containing node pools | list | `` | no | -| node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node_pools_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | -| non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project_id | The project ID to host the cluster in (required) | string | - | yes | -| region | The region to host the cluster in (required) | string | - | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | -| remove_default_node_pool | Remove default node pool while setting up the cluster | string | `false` | no | -| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account | string | `` | no | -| stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | string | `"false"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | Cluster ca certificate (base64 encoded) | +| ca\_certificate | Cluster ca certificate (base64 encoded) | | endpoint | Cluster endpoint | -| horizontal_pod_autoscaling_enabled | Whether horizontal pod autoscaling enabled | -| http_load_balancing_enabled | Whether http load balancing enabled | -| kubernetes_dashboard_enabled | Whether kubernetes dashboard enabled | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging_service | Logging service used | -| master_authorized_networks_config | Networks from which access to master is permitted | -| master_version | Current master kubernetes version | -| min_master_version | Minimum master kubernetes version | -| monitoring_service | Monitoring service used | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | | name | Cluster name | -| network_policy_enabled | Whether network policy enabled | -| node_pools_names | List of node pools names | -| node_pools_versions | List of node pools versions | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | | region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | @@ -171,7 +177,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.8.0 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.0.0 ### Configure a Service Account In order to execute this module you must have a Service Account with the @@ -181,6 +187,7 @@ following project roles: - roles/container.developer - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: @@ -239,6 +246,9 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i 1. Configure the [test fixtures](#test-configuration) 2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. + - Requires the [permissions to run the module](#configure-a-service-account) + - Requires `roles/compute.networkAdmin` to create the test suite's networks + - Requires `roles/resourcemanager.projectIamAdmin` since service account creation is tested 3. Build the Docker container for testing: ``` @@ -261,6 +271,20 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. +If you wish to parallelize running the test suites, it is also possible to offload the work onto Concourse to run each test suite for you using the command `make test_integration_concourse`. The `.concourse` directory will be created and contain all of the logs from the running test suites. + +When running tests locally, you will need to use your own test project environment. You can configure your environment by setting all of the following variables: + +``` +export COMPUTE_ENGINE_SERVICE_ACCOUNT="" +export PROJECT_ID="" +export REGION="" +export ZONES='[""]' +export SERVICE_ACCOUNT_JSON="$(cat "")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" +export GOOGLE_APPLICATION_CREDENTIALS="" +``` + #### Test configuration Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. diff --git a/auth.tf b/auth.tf index 5ad4160145..4ec9fefa18 100644 --- a/auth.tf +++ b/auth.tf @@ -31,4 +31,4 @@ provider "kubernetes" { host = "https://${local.cluster_endpoint}" token = "${data.google_client_config.default.access_token}" cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" -} +} \ No newline at end of file diff --git a/autogen/README.md b/autogen/README.md index 50cda2d5a1..caca32666a 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -95,70 +95,13 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure +## Upgrade to v1.0.0 -[^]: (autogen_docs_start) - +Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| description | The description of the cluster | string | `` | no | -| horizontal_pod_autoscaling | Enable horizontal pod autoscaling addon | string | `true` | no | -| http_load_balancing | Enable httpload balancer addon | string | `true` | no | -| ip_masq_link_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `false` | no | -| ip_masq_resync_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `60s` | no | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no | -| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no | -| logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no | -| maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no | -| master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `monitoring.googleapis.com` | no | -| name | The name of the cluster (required) | string | - | yes | -| network | The VPC network to host the cluster in (required) | string | - | yes | -| network_policy | Enable network policy addon | string | `false` | no | -| network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | -| remove_default_node_pool | Boolean value determining removal of default node pool | bool | false | no | -| node_pools | List of maps containing node pools | list | `` | no | -| node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node_pools_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | -| non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project_id | The project ID to host the cluster in (required) | string | - | yes | -| region | The region to host the cluster in (required) | string | - | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | -| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account | string | `` | no | -| stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal_pod_autoscaling_enabled | Whether horizontal pod autoscaling enabled | -| http_load_balancing_enabled | Whether http load balancing enabled | -| kubernetes_dashboard_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging_service | Logging service used | -| master_authorized_networks_config | Networks from which access to master is permitted | -| master_version | Current master kubernetes version | -| min_master_version | Minimum master kubernetes version | -| monitoring_service | Monitoring service used | -| name | Cluster name | -| network_policy_enabled | Whether network policy enabled | -| node_pools_names | List of node pools names | -| node_pools_versions | List of node pools versions | -| region | Cluster region | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | +In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +[^]: (autogen_docs_start) [^]: (autogen_docs_end) ## Requirements @@ -178,9 +121,9 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x {% if private_cluster %} -- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v1.20.0 +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.0.0 {% else %} -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.8.0 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v2.0.0 {% endif %} ### Configure a Service Account @@ -191,6 +134,7 @@ following project roles: - roles/container.developer - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: @@ -249,6 +193,9 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i 1. Configure the [test fixtures](#test-configuration) 2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. + - Requires the [permissions to run the module](#configure-a-service-account) + - Requires `roles/compute.networkAdmin` to create the test suite's networks + - Requires `roles/resourcemanager.projectIamAdmin` since service account creation is tested 3. Build the Docker container for testing: ``` @@ -271,6 +218,20 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. +If you wish to parallelize running the test suites, it is also possible to offload the work onto Concourse to run each test suite for you using the command `make test_integration_concourse`. The `.concourse` directory will be created and contain all of the logs from the running test suites. + +When running tests locally, you will need to use your own test project environment. You can configure your environment by setting all of the following variables: + +``` +export COMPUTE_ENGINE_SERVICE_ACCOUNT="" +export PROJECT_ID="" +export REGION="" +export ZONES='[""]' +export SERVICE_ACCOUNT_JSON="$(cat "")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" +export GOOGLE_APPLICATION_CREDENTIALS="" +``` + #### Test configuration Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index e7f9f0587f..d4b17b8cf0 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_regional}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -98,13 +98,13 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "google-beta" count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" region = "${var.region}" cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -121,7 +121,7 @@ resource "google_container_node_pool" "pools" { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index a77e7d527c..da99b4b43f 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_zonal}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -98,13 +98,13 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "google-beta" count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" zone = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -121,7 +121,7 @@ resource "google_container_node_pool" "zonal_pools" { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] diff --git a/autogen/main.tf b/autogen/main.tf index e3f405bbdd..c803eedc4f 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}" - node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}" - custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" - network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" + kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}" + kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}" + node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}" + node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}" + custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" + network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" cluster_type = "${var.regional ? "regional" : "zonal"}" @@ -149,7 +151,16 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" - zone = "${data.google_compute_zones.available.names[0]}" + provider = "google-beta" + region = "${var.region}" + project = "${var.project_id}" +} + +data "google_container_engine_versions" "zone" { + // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error + // + // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. + // + zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" project = "${var.project_id}" } diff --git a/autogen/outputs.tf b/autogen/outputs.tf index 839d146f6e..e0977f6df8 100644 --- a/autogen/outputs.tf +++ b/autogen/outputs.tf @@ -107,3 +107,8 @@ output "node_pools_versions" { description = "List of node pools versions" value = "${local.cluster_node_pools_versions}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${local.service_account}" +} diff --git a/autogen/sa.tf b/autogen/sa.tf index 443f408601..56220640dd 100644 --- a/autogen/sa.tf +++ b/autogen/sa.tf @@ -21,10 +21,17 @@ locals { service_account = "${var.service_account == "create" ? element(local.service_account_list, 0) : var.service_account}" } +resource "random_string" "cluster_service_account_suffix" { + upper = "false" + lower = "true" + special = "false" + length = 4 +} + resource "google_service_account" "cluster_service_account" { count = "${var.service_account == "create" ? 1 : 0}" project = "${var.project_id}" - account_id = "tf-gke-${substr(var.name, 0, min(20, length(var.name)))}" + account_id = "tf-gke-${substr(var.name, 0, min(15, length(var.name)))}-${random_string.cluster_service_account_suffix.result}" display_name = "Terraform-managed service account for cluster ${var.name}" } diff --git a/autogen/variables.tf b/autogen/variables.tf index f1ccae2754..7636039b49 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -112,11 +112,11 @@ variable "maintenance_start_time" { } variable "ip_range_pods" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet ip range to use for pods" } variable "ip_range_services" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet range to use for services" } variable "remove_default_node_pool" { @@ -124,6 +124,12 @@ variable "remove_default_node_pool" { default = false } +variable "disable_legacy_metadata_endpoints" { + description = "Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated." + default = "true" +} + + variable "node_pools" { type = "list" description = "List of maps containing node pools" diff --git a/cluster_regional.tf b/cluster_regional.tf index 952d9df0b6..f31877961a 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_regional}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -84,7 +84,6 @@ resource "google_container_cluster" "primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -92,13 +91,13 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google" + provider = "google-beta" count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" region = "${var.region}" cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -115,7 +114,7 @@ resource "google_container_node_pool" "pools" { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] @@ -153,4 +152,4 @@ resource "null_resource" "wait_for_regional_cluster" { } depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] -} +} \ No newline at end of file diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 0d2194761d..aa26cc4fc4 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_zonal}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -84,7 +84,6 @@ resource "google_container_cluster" "zonal_primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -92,13 +91,13 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google" + provider = "google-beta" count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" zone = "${var.zones[0]}" cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -108,14 +107,14 @@ resource "google_container_node_pool" "zonal_pools" { management { auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" } node_config { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] @@ -153,4 +152,4 @@ resource "null_resource" "wait_for_zonal_cluster" { } depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/dns.tf b/dns.tf index 25effe580a..56d702e00a 100644 --- a/dns.tf +++ b/dns.tf @@ -51,4 +51,4 @@ EOF } depends_on = ["null_resource.delete_default_kube_dns_configmap", "data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md index dac502d107..9d2429490d 100644 --- a/examples/deploy_service/README.md +++ b/examples/deploy_service/README.md @@ -10,37 +10,37 @@ It will: [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index 82b824ff45..6e2a3d2d5c 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -19,9 +19,15 @@ locals { } provider "google" { + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" - version = "~> 1.20" } provider "kubernetes" { diff --git a/examples/deploy_service/outputs.tf b/examples/deploy_service/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/deploy_service/outputs.tf +++ b/examples/deploy_service/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index f96aa1bfe5..d3a3ab7098 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -4,37 +4,38 @@ This example illustrates how to create a cluster with multiple custom node-pool [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 14fdd9e3a3..bb5c8c74ed 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -19,23 +19,30 @@ locals { } provider "google" { - version = "~> 1.20" + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } module "gke" { - source = "../../" - project_id = "${var.project_id}" - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = "false" - region = "${var.region}" - zones = "${var.zones}" - network = "${var.network}" - subnetwork = "${var.subnetwork}" - ip_range_pods = "${var.ip_range_pods}" - ip_range_services = "${var.ip_range_services}" - remove_default_node_pool = "true" + source = "../../" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = "false" + region = "${var.region}" + zones = "${var.zones}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + remove_default_node_pool = "true" + disable_legacy_metadata_endpoints = "false" node_pools = [ { @@ -43,6 +50,7 @@ module "gke" { min_count = 1 max_count = 2 service_account = "${var.compute_engine_service_account}" + auto_upgrade = true }, { name = "pool-02" @@ -53,7 +61,6 @@ module "gke" { disk_type = "pd-standard" image_type = "COS" auto_repair = false - auto_upgrade = false service_account = "${var.compute_engine_service_account}" }, ] diff --git a/examples/node_pool/outputs.tf b/examples/node_pool/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/node_pool/outputs.tf +++ b/examples/node_pool/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md index 776f9a7ace..86a561331f 100644 --- a/examples/shared_vpc/README.md +++ b/examples/shared_vpc/README.md @@ -4,38 +4,38 @@ This example illustrates how to create a simple cluster where the host network i [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| network_project_id | The GCP project housing the VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| network\_project\_id | The GCP project housing the VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf index 75877b31a6..d4a4283a6f 100644 --- a/examples/shared_vpc/main.tf +++ b/examples/shared_vpc/main.tf @@ -19,7 +19,13 @@ locals { } provider "google" { - version = "~> 1.20" + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/shared_vpc/outputs.tf b/examples/shared_vpc/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/shared_vpc/outputs.tf +++ b/examples/shared_vpc/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md index 7046eefb64..f37db67dfc 100644 --- a/examples/simple_regional/README.md +++ b/examples/simple_regional/README.md @@ -4,37 +4,37 @@ This example illustrates how to create a simple cluster. [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 5694ace8e4..328658cff8 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -19,6 +19,13 @@ locals { } provider "google" { + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/simple_regional/outputs.tf b/examples/simple_regional/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/simple_regional/outputs.tf +++ b/examples/simple_regional/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md index 431d265446..0b7a1089e1 100644 --- a/examples/simple_regional_private/README.md +++ b/examples/simple_regional_private/README.md @@ -4,37 +4,37 @@ This example illustrates how to create a simple private cluster. [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index 56e9ad598b..1f1776c8a6 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 1.20" + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/simple_regional_private/outputs.tf +++ b/examples/simple_regional_private/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md index 9a1585534b..0e528744dc 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal/README.md @@ -4,37 +4,37 @@ This example illustrates how to create a simple cluster. [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | -| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf index a0dc9ffa97..71fe57ed28 100644 --- a/examples/simple_zonal/main.tf +++ b/examples/simple_zonal/main.tf @@ -19,7 +19,13 @@ locals { } provider "google" { - version = "~> 1.20" + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/simple_zonal/outputs.tf +++ b/examples/simple_zonal/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md index ea13ff1bb5..cc1643aec8 100644 --- a/examples/simple_zonal_private/README.md +++ b/examples/simple_zonal_private/README.md @@ -4,38 +4,38 @@ This example illustrates how to create a simple private cluster. [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | -| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index 351370c3c8..76f7d6da13 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 1.20" + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/simple_zonal_private/outputs.tf +++ b/examples/simple_zonal_private/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md index f5f8c4a455..5bff5843f1 100644 --- a/examples/stub_domains/README.md +++ b/examples/stub_domains/README.md @@ -9,37 +9,37 @@ It will: [^]: (autogen_docs_start) - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | -| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | -| credentials_path | The path to the GCP credentials JSON file | string | - | yes | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| network | The VPC network to host the cluster in | string | - | yes | -| project_id | The project ID to host the cluster in | string | - | yes | -| region | The region to host the cluster in | string | - | yes | -| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | | -| client_token | | -| cluster_name | Cluster name | -| credentials_path | | -| ip_range_pods | The secondary IP range used for pods | -| ip_range_services | The secondary IP range used for services | -| kubernetes_endpoint | | +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | | location | | -| master_kubernetes_version | The master Kubernetes version | +| master\_kubernetes\_version | The master Kubernetes version | | network | | -| project_id | | +| project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index 71dbb19e7e..1a5568e663 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -19,7 +19,13 @@ locals { } provider "google" { - version = "~> 1.20" + version = "~> 2.2" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.2" credentials = "${file(var.credentials_path)}" region = "${var.region}" } diff --git a/examples/stub_domains/outputs.tf b/examples/stub_domains/outputs.tf index b48cab862e..9881585a48 100644 --- a/examples/stub_domains/outputs.tf +++ b/examples/stub_domains/outputs.tf @@ -27,3 +27,8 @@ output "client_token" { output "ca_certificate" { value = "${module.gke.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} diff --git a/main.tf b/main.tf index ad84f1daa3..ca2ae78efd 100644 --- a/main.tf +++ b/main.tf @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}" - node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}" - custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" - network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" + kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}" + kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}" + node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}" + node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}" + custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" + network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" cluster_type = "${var.regional ? "regional" : "zonal"}" @@ -149,7 +151,17 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google" - zone = "${data.google_compute_zones.available.names[0]}" + provider = "google-beta" + region = "${var.region}" project = "${var.project_id}" } + +data "google_container_engine_versions" "zone" { + // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error + // + // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. + // + zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + + project = "${var.project_id}" +} diff --git a/masq.tf b/masq.tf index a78d263cef..36ce6e21a8 100644 --- a/masq.tf +++ b/masq.tf @@ -41,4 +41,4 @@ EOF } depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index f0611c7631..e3eeb194cf 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -92,70 +92,76 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure +## Upgrade to v1.0.0 -[^]: (autogen_docs_start) +Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. + +In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +[^]: (autogen_docs_start) ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| description | The description of the cluster | string | `` | no | -| enable_private_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `false` | no | -| enable_private_nodes | (Beta) Whether nodes have internal IP addresses only | string | `false` | no | -| horizontal_pod_autoscaling | Enable horizontal pod autoscaling addon | string | `true` | no | -| http_load_balancing | Enable httpload balancer addon | string | `true` | no | -| ip_masq_link_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `false` | no | -| ip_masq_resync_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `60s` | no | -| ip_range_pods | The secondary ip range to use for pods | string | - | yes | -| ip_range_services | The secondary ip range to use for pods | string | - | yes | -| kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no | -| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no | -| logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no | -| maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no | -| master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| master_ipv4_cidr_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `10.0.0.0/28` | no | -| monitoring_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `monitoring.googleapis.com` | no | -| name | The name of the cluster (required) | string | - | yes | -| network | The VPC network to host the cluster in (required) | string | - | yes | -| network_policy | Enable network policy addon | string | `false` | no | -| network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | -| node_pools | List of maps containing node pools | list | `` | no | -| node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node_pools_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | -| non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project_id | The project ID to host the cluster in (required) | string | - | yes | -| region | The region to host the cluster in (required) | string | - | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | -| remove_default_node_pool | Remove default node pool while setting up the cluster | string | `false` | no | -| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account | string | `` | no | -| stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | string | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | string | `"false"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | ## Outputs | Name | Description | |------|-------------| -| ca_certificate | Cluster ca certificate (base64 encoded) | +| ca\_certificate | Cluster ca certificate (base64 encoded) | | endpoint | Cluster endpoint | -| horizontal_pod_autoscaling_enabled | Whether horizontal pod autoscaling enabled | -| http_load_balancing_enabled | Whether http load balancing enabled | -| kubernetes_dashboard_enabled | Whether kubernetes dashboard enabled | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging_service | Logging service used | -| master_authorized_networks_config | Networks from which access to master is permitted | -| master_version | Current master kubernetes version | -| min_master_version | Minimum master kubernetes version | -| monitoring_service | Monitoring service used | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | | name | Cluster name | -| network_policy_enabled | Whether network policy enabled | -| node_pools_names | List of node pools names | -| node_pools_versions | List of node pools versions | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | | region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | @@ -177,7 +183,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.11.x -- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v1.20.0 +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v2.0.0 ### Configure a Service Account In order to execute this module you must have a Service Account with the @@ -187,6 +193,7 @@ following project roles: - roles/container.developer - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: @@ -245,6 +252,9 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i 1. Configure the [test fixtures](#test-configuration) 2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. + - Requires the [permissions to run the module](#configure-a-service-account) + - Requires `roles/compute.networkAdmin` to create the test suite's networks + - Requires `roles/resourcemanager.projectIamAdmin` since service account creation is tested 3. Build the Docker container for testing: ``` @@ -267,6 +277,20 @@ The test-kitchen instances in `test/fixtures/` wrap identically-named examples i Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. +If you wish to parallelize running the test suites, it is also possible to offload the work onto Concourse to run each test suite for you using the command `make test_integration_concourse`. The `.concourse` directory will be created and contain all of the logs from the running test suites. + +When running tests locally, you will need to use your own test project environment. You can configure your environment by setting all of the following variables: + +``` +export COMPUTE_ENGINE_SERVICE_ACCOUNT="" +export PROJECT_ID="" +export REGION="" +export ZONES='[""]' +export SERVICE_ACCOUNT_JSON="$(cat "")" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" +export GOOGLE_APPLICATION_CREDENTIALS="" +``` + #### Test configuration Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. diff --git a/modules/private-cluster/auth.tf b/modules/private-cluster/auth.tf index 0bbafaf4a2..56e015b1ef 100644 --- a/modules/private-cluster/auth.tf +++ b/modules/private-cluster/auth.tf @@ -31,4 +31,4 @@ provider "kubernetes" { host = "https://${local.cluster_endpoint}" token = "${data.google_client_config.default.access_token}" cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" -} +} \ No newline at end of file diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index b6a77059f0..08f960f9d3 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_regional}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -84,13 +84,11 @@ resource "google_container_cluster" "primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } - private_cluster_config { enable_private_endpoint = "${var.enable_private_endpoint}" enable_private_nodes = "${var.enable_private_nodes}" master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -104,7 +102,7 @@ resource "google_container_node_pool" "pools" { project = "${var.project_id}" region = "${var.region}" cluster = "${var.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -121,7 +119,7 @@ resource "google_container_node_pool" "pools" { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] @@ -161,4 +159,4 @@ resource "null_resource" "wait_for_regional_cluster" { } depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] -} +} \ No newline at end of file diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index 51269e6a61..4675f5d6aa 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" { network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version}" + min_master_version = "${local.kubernetes_version_zonal}" logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" @@ -84,13 +84,11 @@ resource "google_container_cluster" "zonal_primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } - private_cluster_config { enable_private_endpoint = "${var.enable_private_endpoint}" enable_private_nodes = "${var.enable_private_nodes}" master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -104,7 +102,7 @@ resource "google_container_node_pool" "zonal_pools" { project = "${var.project_id}" zone = "${var.zones[0]}" cluster = "${var.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" autoscaling { @@ -121,7 +119,7 @@ resource "google_container_node_pool" "zonal_pools" { image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] @@ -161,4 +159,4 @@ resource "null_resource" "wait_for_zonal_cluster" { } depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 25effe580a..56d702e00a 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -51,4 +51,4 @@ EOF } depends_on = ["null_resource.delete_default_kube_dns_configmap", "data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 57d98dd2b0..3e93de103c 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}" - node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}" - custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" - network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" + kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}" + kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}" + node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}" + node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}" + custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" + network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" cluster_type = "${var.regional ? "regional" : "zonal"}" @@ -150,6 +152,16 @@ locals { *****************************************/ data "google_container_engine_versions" "region" { provider = "google-beta" - zone = "${data.google_compute_zones.available.names[0]}" + region = "${var.region}" project = "${var.project_id}" } + +data "google_container_engine_versions" "zone" { + // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error + // + // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. + // + zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + + project = "${var.project_id}" +} diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index a78d263cef..36ce6e21a8 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -41,4 +41,4 @@ EOF } depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} +} \ No newline at end of file diff --git a/modules/private-cluster/networks.tf b/modules/private-cluster/networks.tf index aeb1a99aba..9d8f7394c2 100644 --- a/modules/private-cluster/networks.tf +++ b/modules/private-cluster/networks.tf @@ -27,4 +27,4 @@ data "google_compute_subnetwork" "gke_subnetwork" { name = "${var.subnetwork}" region = "${var.region}" project = "${local.network_project_id}" -} +} \ No newline at end of file diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 984530fc42..455f69790a 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -107,3 +107,8 @@ output "node_pools_versions" { description = "List of node pools versions" value = "${local.cluster_node_pools_versions}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${local.service_account}" +} \ No newline at end of file diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index f766f0475d..0b119e41f6 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -21,10 +21,17 @@ locals { service_account = "${var.service_account == "create" ? element(local.service_account_list, 0) : var.service_account}" } +resource "random_string" "cluster_service_account_suffix" { + upper = "false" + lower = "true" + special = "false" + length = 4 +} + resource "google_service_account" "cluster_service_account" { count = "${var.service_account == "create" ? 1 : 0}" project = "${var.project_id}" - account_id = "tf-gke-${substr(var.name, 0, min(20, length(var.name)))}" + account_id = "tf-gke-${substr(var.name, 0, min(15, length(var.name)))}-${random_string.cluster_service_account_suffix.result}" display_name = "Terraform-managed service account for cluster ${var.name}" } @@ -47,4 +54,4 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" project = "${google_project_iam_member.cluster_service_account-metric_writer.project}" role = "roles/monitoring.viewer" member = "serviceAccount:${google_service_account.cluster_service_account.email}" -} +} \ No newline at end of file diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 448832cd57..5bef1d19cf 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -112,11 +112,11 @@ variable "maintenance_start_time" { } variable "ip_range_pods" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet ip range to use for pods" } variable "ip_range_services" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet range to use for services" } variable "remove_default_node_pool" { @@ -124,6 +124,12 @@ variable "remove_default_node_pool" { default = false } +variable "disable_legacy_metadata_endpoints" { + description = "Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated." + default = "true" +} + + variable "node_pools" { type = "list" description = "List of maps containing node pools" @@ -211,18 +217,17 @@ variable "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account" default = "" } - variable "enable_private_endpoint" { - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" - default = false + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + default = false } variable "enable_private_nodes" { - description = "(Beta) Whether nodes have internal IP addresses only" - default = false + description = "(Beta) Whether nodes have internal IP addresses only" + default = false } variable "master_ipv4_cidr_block" { - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" - default = "10.0.0.0/28" -} + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} \ No newline at end of file diff --git a/networks.tf b/networks.tf index f6c26d9e77..f1cd380a73 100644 --- a/networks.tf +++ b/networks.tf @@ -27,4 +27,4 @@ data "google_compute_subnetwork" "gke_subnetwork" { name = "${var.subnetwork}" region = "${var.region}" project = "${local.network_project_id}" -} +} \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index 984530fc42..455f69790a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -107,3 +107,8 @@ output "node_pools_versions" { description = "List of node pools versions" value = "${local.cluster_node_pools_versions}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${local.service_account}" +} \ No newline at end of file diff --git a/sa.tf b/sa.tf index f766f0475d..0b119e41f6 100644 --- a/sa.tf +++ b/sa.tf @@ -21,10 +21,17 @@ locals { service_account = "${var.service_account == "create" ? element(local.service_account_list, 0) : var.service_account}" } +resource "random_string" "cluster_service_account_suffix" { + upper = "false" + lower = "true" + special = "false" + length = 4 +} + resource "google_service_account" "cluster_service_account" { count = "${var.service_account == "create" ? 1 : 0}" project = "${var.project_id}" - account_id = "tf-gke-${substr(var.name, 0, min(20, length(var.name)))}" + account_id = "tf-gke-${substr(var.name, 0, min(15, length(var.name)))}-${random_string.cluster_service_account_suffix.result}" display_name = "Terraform-managed service account for cluster ${var.name}" } @@ -47,4 +54,4 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" project = "${google_project_iam_member.cluster_service_account-metric_writer.project}" role = "roles/monitoring.viewer" member = "serviceAccount:${google_service_account.cluster_service_account.email}" -} +} \ No newline at end of file diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 3af8e16983..ee85f31f4e 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -77,3 +77,8 @@ output "ca_certificate" { description = "The cluster CA certificate" value = "${module.example.ca_certificate}" } + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.example.service_account}" +} diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 578c5d98ab..a89e3eced0 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -111,6 +111,7 @@ "config" => including( "metadata" => including( "shutdown-script" => File.open("examples/node_pool/data/shutdown-script.sh").read, + "disable-legacy-endpoints" => "false", ), ), ) diff --git a/test/integration/simple_zonal/controls/gcloud.rb b/test/integration/simple_zonal/controls/gcloud.rb index ce11017228..f65986b0a8 100644 --- a/test/integration/simple_zonal/controls/gcloud.rb +++ b/test/integration/simple_zonal/controls/gcloud.rb @@ -15,6 +15,7 @@ project_id = attribute('project_id') location = attribute('location') cluster_name = attribute('cluster_name') +service_account = attribute('service_account') credentials_path = attribute('credentials_path') ENV['CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE'] = credentials_path @@ -83,7 +84,7 @@ expect(node_pools).to include( including( "config" => including( - "serviceAccount" => starting_with("tf-gke-simple-zonal-cluster@"), + "serviceAccount" => service_account, ), ), ) diff --git a/test/integration/simple_zonal/inspec.yml b/test/integration/simple_zonal/inspec.yml index d17e1cceef..8b66718e9c 100644 --- a/test/integration/simple_zonal/inspec.yml +++ b/test/integration/simple_zonal/inspec.yml @@ -21,3 +21,6 @@ attributes: - name: client_token required: true type: string + - name: service_account + required: true + type: string diff --git a/test/make.sh b/test/make.sh index 095b21b889..6ad9585c54 100755 --- a/test/make.sh +++ b/test/make.sh @@ -18,14 +18,12 @@ # shebang has a '- e' flag, which causes it # to exit on error function check_bash() { -find . -name "*.sh" | while IFS= read -d '' -r file; -do - if [[ "$file" != *"bash -e"* ]]; - then - echo "$file is missing shebang with -e"; - exit 1; - fi; -done; + find . -name "*.sh" | while IFS= read -d '' -r file; do + if [[ "$file" != *"bash -e"* ]]; then + echo "$file is missing shebang with -e" + exit 1 + fi + done } # This function makes sure that the required files for @@ -92,8 +90,8 @@ function generate_docs() { echo "Generating markdown docs with terraform-docs" TMPFILE=$(mktemp) #shellcheck disable=2006,2086 - for j in `for i in $(find . -type f | grep \.tf$) ; do dirname $i ; done | sort -u` ; do - terraform-docs markdown "$j" > "$TMPFILE" + for j in $(find ./ -name '*.tf' -type f -exec dirname '{}' \; | sort -u | grep -v ./autogen); do + terraform-docs markdown "$j" >"$TMPFILE" python helpers/combine_docfiles.py "$j"/README.md "$TMPFILE" done rm -f "$TMPFILE" diff --git a/variables.tf b/variables.tf index e4f7cac1c3..0977e831ad 100644 --- a/variables.tf +++ b/variables.tf @@ -112,11 +112,11 @@ variable "maintenance_start_time" { } variable "ip_range_pods" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet ip range to use for pods" } variable "ip_range_services" { - description = "The secondary ip range to use for pods" + description = "The _name_ of the secondary subnet range to use for services" } variable "remove_default_node_pool" { @@ -124,6 +124,12 @@ variable "remove_default_node_pool" { default = false } +variable "disable_legacy_metadata_endpoints" { + description = "Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated." + default = "true" +} + + variable "node_pools" { type = "list" description = "List of maps containing node pools" @@ -210,4 +216,4 @@ variable "monitoring_service" { variable "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account. May also specify `create` to automatically create a cluster-specific service account" default = "" -} +} \ No newline at end of file