diff --git a/.kitchen.yml b/.kitchen.yml index 8229f01f8b..0c17e3c5e9 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -76,6 +76,19 @@ suites: backend: local provisioner: name: terraform + - name: "simple_regional_private" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/simple_regional_private + verifier: + name: terraform + color: false + systems: + - name: simple_regional_private + backend: local + provisioner: + name: terraform - name: "simple_zonal" driver: name: "terraform" @@ -89,6 +102,19 @@ suites: backend: local provisioner: name: terraform + - name: "simple_zonal_private" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: test/fixtures/simple_zonal_private + verifier: + name: terraform + color: false + systems: + - name: simple_zonal_private + backend: local + provisioner: + name: terraform - name: "stub_domains" driver: name: "terraform" diff --git a/CHANGELOG.md b/CHANGELOG.md index af0547a148..714e2df341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Changed +* Add support for private clusters via submodule. #69 * Set `horizontal_pod_autoscaling` to `true` by default. #42 * Add `remove_default_node_pool` set to `false` by default #15 * Allow arbitrary key-value pairs to be set on node pool metadata. #52 -* Add `initial_node_count` parameter to node_pool block. #60 +* Add `initial_node_count` parameter to node_pool block. #60 ## [v0.4.0] - 2018-12-19 ### Added diff --git a/README.md b/README.md index daa5741f2a..b3913a8b65 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Terraform Kubernetes Engine Module This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. - The resources/services/activations/deletions that this module will create/trigger are: - Create a GKE cluster with the provided addons - Create GKE Node Pool(s) with provided configuration and attach to cluster @@ -115,7 +114,6 @@ Then perform the following commands on the root folder: | network | The VPC network to host the cluster in (required) | string | - | yes | | network_policy | Enable network policy addon | string | `false` | no | | network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | -| remove_default_node_pool | Boolean value determining removal of default node pool | bool | false | no | | node_pools | List of maps containing node pools | list | `` | no | | node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | | node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | @@ -126,6 +124,7 @@ Then perform the following commands on the root folder: | project_id | The project ID to host the cluster in (required) | string | - | yes | | region | The region to host the cluster in (required) | string | - | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | +| remove_default_node_pool | Remove default node pool while setting up the cluster | string | `false` | no | | service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account | string | `` | no | | stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | @@ -168,11 +167,11 @@ Before this module can be used on a project, you must ensure that the following The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. ### Software Dependencies -### Kubectl +#### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x -### Terraform plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.10.x -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.8.0 +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.8.0 ### Configure a Service Account In order to execute this module you must have a Service Account with the @@ -188,12 +187,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## Install - -### Terraform -Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: -- https://releases.hashicorp.com/terraform/ - ## File structure The project has the following folders and files: @@ -209,9 +202,9 @@ The project has the following folders and files: ## Templating -To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](./autogen) directory. +To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](/autogen) directory. -The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](./autogen) directory where appropriate. +The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate. ## Testing @@ -270,7 +263,7 @@ Alternatively, you can simply run `make test_integration_docker` to run all the #### Test configuration Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. -For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. +For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed. diff --git a/auth.tf b/auth.tf index b32b9e0343..5ad4160145 100644 --- a/auth.tf +++ b/auth.tf @@ -19,7 +19,9 @@ /****************************************** Retrieve authentication token *****************************************/ -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google" +} /****************************************** Configure provider @@ -29,4 +31,4 @@ provider "kubernetes" { host = "https://${local.cluster_endpoint}" token = "${data.google_client_config.default.access_token}" cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" -} \ No newline at end of file +} diff --git a/autogen/README.md b/autogen/README.md new file mode 100644 index 0000000000..5f5c9123ed --- /dev/null +++ b/autogen/README.md @@ -0,0 +1,326 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. {% if private_cluster %}This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters){% endif %} + +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `network_policy` is true + +## Usage +There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: + +```hcl +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google{% if private_cluster %}//modules/private-cluster{% endif %}" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + http_load_balancing = false + horizontal_pod_autoscaling = true + kubernetes_dashboard = true + network_policy = true + {% if private_cluster %} + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "10.0.0.0/28" + {% endif %} + + node_pools = [ + { + name = "default-node-pool" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 100 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + }, + ] + + node_pools_labels = { + all = {} + + default-node-pool = { + default-node-pool = "true" + } + } + + node_pools_metadata = { + all = {} + + default-node-pool = { + node-pool-metadata-custom-value = "my-node-pool" + } + } + + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = "true" + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + + node_pools_tags = { + all = [] + + default-node-pool = [ + "default-node-pool", + ] + } +} +``` + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| description | The description of the cluster | string | `` | no | +| horizontal_pod_autoscaling | Enable horizontal pod autoscaling addon | string | `true` | no | +| http_load_balancing | Enable httpload balancer addon | string | `true` | no | +| ip_masq_link_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `false` | no | +| ip_masq_resync_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `60s` | no | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no | +| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no | +| logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no | +| maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no | +| master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| monitoring_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `monitoring.googleapis.com` | no | +| name | The name of the cluster (required) | string | - | yes | +| network | The VPC network to host the cluster in (required) | string | - | yes | +| network_policy | Enable network policy addon | string | `false` | no | +| network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | +| remove_default_node_pool | Boolean value determining removal of default node pool | bool | false | no | +| node_pools | List of maps containing node pools | list | `` | no | +| node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node_pools_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | +| non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project_id | The project ID to host the cluster in (required) | string | - | yes | +| region | The region to host the cluster in (required) | string | - | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | +| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account | string | `` | no | +| stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal_pod_autoscaling_enabled | Whether horizontal pod autoscaling enabled | +| http_load_balancing_enabled | Whether http load balancing enabled | +| kubernetes_dashboard_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging_service | Logging service used | +| master_authorized_networks_config | Networks from which access to master is permitted | +| master_version | Current master kubernetes version | +| min_master_version | Minimum master kubernetes version | +| monitoring_service | Monitoring service used | +| name | Cluster name | +| network_policy_enabled | Whether network policy enabled | +| node_pools_names | List of node pools names | +| node_pools_versions | List of node pools versions | +| region | Cluster region | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +{% if private_cluster %} +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v1.20.0 +{% else %} +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) v1.8.0 +{% endif %} + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountUser + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +## File structure +The project has the following folders and files: + +- /: root folder +- /examples: examples for using this module +- /helpers: Helper scripts +- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file) +- /test: Folders with files for testing the module (see Testing section on this file) +- /main.tf: main file for this module, contains all the resources to create +- /variables.tf: all the variables for the module +- /output.tf: the outputs of the module +- /readme.MD: this file + +## Templating + +To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](/autogen) directory. + +The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate. + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + +Six test-kitchen instances are defined: + +- `deploy-service` +- `node-pool` +- `shared-vpc` +- `simple-regional` +- `simple-zonal` +- `stub-domains` + +The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory. + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. +3. Build the Docker container for testing: + + ``` + make docker_build_kitchen_terraform + ``` +4. Run the testing container in interactive mode: + + ``` + make docker_run + ``` + + The module root directory will be loaded into the Docker container at `/cft/workdir/`. +5. Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. Run `kitchen converge` again. This is necessary due to an oddity in how `networkPolicyConfig` is handled by the upstream API. (See [#72](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/72) for details). + 4. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 5. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. + +#### Test configuration + +Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. +For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. +Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. + +Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running hadolint on Dockerfiles +Checking for required files +Testing the validity of the header check +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.026s + +OK +Checking file headers +The following lines have trailing whitespace +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Python - flake8. Can be installed with 'pip install flake8' +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew diff --git a/autogen/auth.tf b/autogen/auth.tf index a64e83fe87..3e961cd6b1 100644 --- a/autogen/auth.tf +++ b/autogen/auth.tf @@ -19,7 +19,9 @@ /****************************************** Retrieve authentication token *****************************************/ -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" +} /****************************************** Configure provider diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 074114af0f..240f23e797 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -20,6 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -35,7 +36,7 @@ resource "google_container_cluster" "primary" { logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" - master_authorized_networks_config = "${var.master_authorized_networks_config}" + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] addons_config { http_load_balancing { @@ -83,7 +84,13 @@ resource "google_container_cluster" "primary" { service_account = "${lookup(var.node_pools[0], "service_account", var.service_account)}" } } - +{% if private_cluster %} + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } +{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -91,6 +98,7 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 7f66c32c9c..f56fde248c 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -20,6 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -35,7 +36,7 @@ resource "google_container_cluster" "zonal_primary" { logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" - master_authorized_networks_config = "${var.master_authorized_networks_config}" + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] addons_config { http_load_balancing { @@ -83,7 +84,13 @@ resource "google_container_cluster" "zonal_primary" { service_account = "${lookup(var.node_pools[0], "service_account", var.service_account)}" } } - +{% if private_cluster %} + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } +{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -91,6 +98,7 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" diff --git a/autogen/main.tf b/autogen/main.tf index f1daf78aee..e3f405bbdd 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -20,8 +20,9 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - project = "${var.project_id}" - region = "${var.region}" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + project = "${var.project_id}" + region = "${var.region}" } resource "random_shuffle" "available_zones" { @@ -148,6 +149,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - zone = "${data.google_compute_zones.available.names[0]}" - project = "${var.project_id}" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + zone = "${data.google_compute_zones.available.names[0]}" + project = "${var.project_id}" } diff --git a/autogen/networks.tf b/autogen/networks.tf index e07589b04d..00d41aa4d0 100644 --- a/autogen/networks.tf +++ b/autogen/networks.tf @@ -17,12 +17,14 @@ {{ autogeneration_note }} data "google_compute_network" "gke_network" { - name = "${var.network}" - project = "${local.network_project_id}" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + name = "${var.network}" + project = "${local.network_project_id}" } data "google_compute_subnetwork" "gke_subnetwork" { - name = "${var.subnetwork}" - region = "${var.region}" - project = "${local.network_project_id}" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + name = "${var.subnetwork}" + region = "${var.region}" + project = "${local.network_project_id}" } diff --git a/autogen/scripts/delete-default-resource.sh b/autogen/scripts/delete-default-resource.sh new file mode 100755 index 0000000000..32b2f51a41 --- /dev/null +++ b/autogen/scripts/delete-default-resource.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -ne 3 ]; then + >&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" -o json "${RESOURCE_NAME}" | jq -r '.metadata.labels."maintained_by"') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi diff --git a/autogen/scripts/kubectl_wrapper.sh b/autogen/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..e92300bcb5 --- /dev/null +++ b/autogen/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" diff --git a/autogen/scripts/wait-for-cluster.sh b/autogen/scripts/wait-for-cluster.sh new file mode 100755 index 0000000000..6ff3253d58 --- /dev/null +++ b/autogen/scripts/wait-for-cluster.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [[ "${current_status}" == "RECONCILING" ]]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" diff --git a/autogen/variables.tf b/autogen/variables.tf index 10fe299009..ab4cd5470a 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -211,3 +211,19 @@ variable "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account" default = "" } +{% if private_cluster %} +variable "enable_private_endpoint" { + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + default = false +} + +variable "enable_private_nodes" { + description = "(Beta) Whether nodes have internal IP addresses only" + default = false +} + +variable "master_ipv4_cidr_block" { + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} +{% endif %} diff --git a/build/docker/kitchen_terraform/Dockerfile b/build/docker/kitchen_terraform/Dockerfile index 4bc42b491c..7be34ac19b 100644 --- a/build/docker/kitchen_terraform/Dockerfile +++ b/build/docker/kitchen_terraform/Dockerfile @@ -14,6 +14,7 @@ ARG BASE_IMAGE +# hadolint ignore=DL3006 FROM $BASE_IMAGE RUN apk add --no-cache \ diff --git a/cluster_regional.tf b/cluster_regional.tf index 2ad8bd4ae9..774b0ee102 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -20,6 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { + provider = "google" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -35,7 +36,7 @@ resource "google_container_cluster" "primary" { logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" - master_authorized_networks_config = "${var.master_authorized_networks_config}" + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] addons_config { http_load_balancing { @@ -91,6 +92,7 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { + provider = "google" count = "${var.regional ? length(var.node_pools) : 0}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" @@ -153,4 +155,4 @@ resource "null_resource" "wait_for_regional_cluster" { } depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] -} \ No newline at end of file +} diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 6dac84e594..c56fa348d5 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -20,6 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { + provider = "google" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -35,7 +36,7 @@ resource "google_container_cluster" "zonal_primary" { logging_service = "${var.logging_service}" monitoring_service = "${var.monitoring_service}" - master_authorized_networks_config = "${var.master_authorized_networks_config}" + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] addons_config { http_load_balancing { @@ -91,6 +92,7 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { + provider = "google" count = "${var.regional ? 0 : length(var.node_pools)}" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" @@ -153,4 +155,4 @@ resource "null_resource" "wait_for_zonal_cluster" { } depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} \ No newline at end of file +} diff --git a/dns.tf b/dns.tf index 56d702e00a..25effe580a 100644 --- a/dns.tf +++ b/dns.tf @@ -51,4 +51,4 @@ EOF } depends_on = ["null_resource.delete_default_kube_dns_configmap", "data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} \ No newline at end of file +} diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md index b0c8748a44..dac502d107 100644 --- a/examples/deploy_service/README.md +++ b/examples/deploy_service/README.md @@ -15,6 +15,7 @@ It will: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | | compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | | credentials_path | The path to the GCP credentials JSON file | string | - | yes | | ip_range_pods | The secondary ip range to use for pods | string | - | yes | @@ -49,4 +50,4 @@ To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 69bb56430b..f96aa1bfe5 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -9,6 +9,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | | compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | | credentials_path | The path to the GCP credentials JSON file | string | - | yes | | ip_range_pods | The secondary ip range to use for pods | string | - | yes | @@ -43,4 +44,4 @@ To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md index 1ecd805d6e..776f9a7ace 100644 --- a/examples/shared_vpc/README.md +++ b/examples/shared_vpc/README.md @@ -4,6 +4,41 @@ This example illustrates how to create a simple cluster where the host network i [^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | +| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | +| credentials_path | The path to the GCP credentials JSON file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| network_project_id | The GCP project housing the VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| credentials_path | | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| master_kubernetes_version | The master Kubernetes version | +| network | | +| project_id | | +| region | | +| subnetwork | | +| zones | List of zones in which the cluster resides | + [^]: (autogen_docs_end) To provision this example, run the following from within this directory: diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md index 7995bde32f..7046eefb64 100644 --- a/examples/simple_regional/README.md +++ b/examples/simple_regional/README.md @@ -9,6 +9,7 @@ This example illustrates how to create a simple cluster. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | | compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | | credentials_path | The path to the GCP credentials JSON file | string | - | yes | | ip_range_pods | The secondary ip range to use for pods | string | - | yes | @@ -43,4 +44,4 @@ To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md new file mode 100644 index 0000000000..431d265446 --- /dev/null +++ b/examples/simple_regional_private/README.md @@ -0,0 +1,47 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple private cluster. + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | +| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | +| credentials_path | The path to the GCP credentials JSON file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| credentials_path | | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| master_kubernetes_version | The master Kubernetes version | +| network | | +| project_id | | +| region | | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf new file mode 100644 index 0000000000..cc1709562f --- /dev/null +++ b/examples/simple_regional_private/main.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-private" +} + +provider "google-beta" { + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +data "google_compute_subnetwork" "subnetwork" { + name = "${var.subnetwork}" + project = "${var.project_id}" + region = "${var.region}" +} + +module "gke" { + source = "../../modules/private-cluster/" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = "${var.region}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "${var.compute_engine_service_account}" + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks_config = [{ + cidr_blocks = [{ + cidr_block = "${data.google_compute_subnetwork.subnetwork.ip_cidr_range}" + display_name = "VPC" + }] + }] +} + +data "google_client_config" "default" {} diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf new file mode 100644 index 0000000000..b48cab862e --- /dev/null +++ b/examples/simple_regional_private/outputs.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + value = "${module.gke.ca_certificate}" +} diff --git a/examples/simple_regional_private/test_outputs.tf b/examples/simple_regional_private/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/simple_regional_private/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/simple_regional_private/variables.tf b/examples/simple_regional_private/variables.tf new file mode 100644 index 0000000000..8501b205b3 --- /dev/null +++ b/examples/simple_regional_private/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to the GCP credentials JSON file" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md index 8653781280..edd120aa05 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal/README.md @@ -9,6 +9,7 @@ This example illustrates how to create a simple cluster. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | | compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | | credentials_path | The path to the GCP credentials JSON file | string | - | yes | | ip_range_pods | The secondary ip range to use for pods | string | - | yes | @@ -44,4 +45,4 @@ To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf index 3f8be1d478..a837cba357 100644 --- a/examples/simple_zonal/main.tf +++ b/examples/simple_zonal/main.tf @@ -24,17 +24,17 @@ provider "google" { } module "gke" { - source = "../../" - project_id = "${var.project_id}" - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = false - region = "${var.region}" - zones = "${var.zones}" - network = "${var.network}" - subnetwork = "${var.subnetwork}" - ip_range_pods = "${var.ip_range_pods}" - ip_range_services = "${var.ip_range_services}" - service_account = "${var.compute_engine_service_account}" + source = "../../" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = false + region = "${var.region}" + zones = "${var.zones}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "${var.compute_engine_service_account}" } data "google_client_config" "default" {} diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md new file mode 100644 index 0000000000..ea13ff1bb5 --- /dev/null +++ b/examples/simple_zonal_private/README.md @@ -0,0 +1,48 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple private cluster. + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | +| compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | +| credentials_path | The path to the GCP credentials JSON file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| credentials_path | | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| master_kubernetes_version | The master Kubernetes version | +| network | | +| project_id | | +| region | | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf new file mode 100644 index 0000000000..f3c3a3ee83 --- /dev/null +++ b/examples/simple_zonal_private/main.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-private" +} + +provider "google-beta" { + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +data "google_compute_subnetwork" "subnetwork" { + name = "${var.subnetwork}" + project = "${var.project_id}" + region = "${var.region}" +} + +module "gke" { + source = "../../modules/private-cluster/" + project_id = "${var.project_id}" + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = false + region = "${var.region}" + zones = "${var.zones}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + service_account = "${var.compute_engine_service_account}" + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks_config = [{ + cidr_blocks = [{ + cidr_block = "${data.google_compute_subnetwork.subnetwork.ip_cidr_range}" + display_name = "VPC" + }] + }] +} + +data "google_client_config" "default" {} diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf new file mode 100644 index 0000000000..b48cab862e --- /dev/null +++ b/examples/simple_zonal_private/outputs.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + value = "${module.gke.ca_certificate}" +} diff --git a/examples/simple_zonal_private/test_outputs.tf b/examples/simple_zonal_private/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/simple_zonal_private/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/simple_zonal_private/variables.tf b/examples/simple_zonal_private/variables.tf new file mode 100644 index 0000000000..ebb151e38a --- /dev/null +++ b/examples/simple_zonal_private/variables.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to the GCP credentials JSON file" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "zones" { + type = "list" + description = "The zone to host the cluster in (required if is a zonal cluster)" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md index fa22bf3f69..f5f8c4a455 100644 --- a/examples/stub_domains/README.md +++ b/examples/stub_domains/README.md @@ -14,6 +14,7 @@ It will: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster_name_suffix | A suffix to append to the default cluster name | string | `` | no | | compute_engine_service_account | Service account to associate to the nodes in the cluster | string | - | yes | | credentials_path | The path to the GCP credentials JSON file | string | - | yes | | ip_range_pods | The secondary ip range to use for pods | string | - | yes | @@ -48,4 +49,4 @@ To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file +- `terraform destroy` to destroy the built infrastructure diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py index 16516cb009..8f13487efc 100644 --- a/helpers/combine_docfiles.py +++ b/helpers/combine_docfiles.py @@ -29,8 +29,8 @@ import re import sys -insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' # noqa: E501 -exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' # noqa: E501 +insert_separator_regex = r'(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' # noqa: E501 +exclude_separator_regex = r'(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' # noqa: E501 if len(sys.argv) != 3: sys.exit(1) @@ -42,14 +42,17 @@ replace_content = open(sys.argv[2], "r").read() # Exclude the specified content from the replacement content -groups = re.match( +matched = re.match( exclude_separator_regex, replace_content, re.DOTALL -).groups(0) -replace_content = groups[0] + groups[1] +) -# Find where to put the replacement content, overwrite the input file -groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0) -output = groups[0] + replace_content + groups[2] -open(sys.argv[1], "w").write(output) +if matched: + groups = matched.groups(0) + replace_content = groups[0] + groups[1] + + # Find where to put the replacement content, overwrite the input file + groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0) + output = groups[0] + replace_content + groups[2] + "\n" + open(sys.argv[1], "w").write(output) diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index 0cdb993bfa..b6e562b5f2 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -21,29 +21,66 @@ from jinja2 import Environment, FileSystemLoader TEMPLATE_FOLDER = "./autogen" +BASE_TEMPLATE_OPTIONS = { + 'autogeneration_note': '// This file was automatically generated ' + + 'from a template in {folder}'.format( + folder=TEMPLATE_FOLDER + ), +} + + +class Module(object): + path = None + options = {} + + def __init__(self, path, template_options): + self.path = path + self.options = template_options + + def template_options(self, base): + return {k: v for d in [base, self.options] for k, v in d.items()} + + +MODULES = [ + Module("./", { + 'private_cluster': False, + }), + Module("./modules/private-cluster", { + 'private_cluster': True, + }), +] +DEVNULL_FILE = open(os.devnull, 'w') def main(argv): env = Environment( loader=FileSystemLoader(TEMPLATE_FOLDER), + trim_blocks=True, + lstrip_blocks=True, ) - template_options = { - 'autogeneration_note': '// This file was automatically generated ' + - 'from a template in {folder}'.format( - folder=TEMPLATE_FOLDER - ), - } templates = env.list_templates() for template_file in templates: - template = env.get_template(template_file) - rendered = template.render(template_options) - with open(os.path.join("./", template_file), "w") as f: - f.write(rendered) - subprocess.call([ - "terraform", - "fmt", - os.path.join("./", template_file) - ]) + for module in MODULES: + template = env.get_template(template_file) + rendered = template.render( + module.template_options(BASE_TEMPLATE_OPTIONS) + ) + with open(os.path.join(module.path, template_file), "w") as f: + f.write(rendered.rstrip()) + if template_file.endswith(".tf"): + subprocess.call( + [ + "terraform", + "fmt", + "-write=true", + os.path.join(module.path, template_file) + ], + stdout=DEVNULL_FILE, + stderr=subprocess.STDOUT + ) + if template_file.endswith(".sh"): + os.chmod(os.path.join(module.path, template_file), 0o755) + DEVNULL_FILE.close() if __name__ == "__main__": diff --git a/main.tf b/main.tf index 8048fff5e8..ad84f1daa3 100644 --- a/main.tf +++ b/main.tf @@ -20,8 +20,9 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - project = "${var.project_id}" - region = "${var.region}" + provider = "google" + project = "${var.project_id}" + region = "${var.region}" } resource "random_shuffle" "available_zones" { @@ -148,6 +149,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - zone = "${data.google_compute_zones.available.names[0]}" - project = "${var.project_id}" -} \ No newline at end of file + provider = "google" + zone = "${data.google_compute_zones.available.names[0]}" + project = "${var.project_id}" +} diff --git a/masq.tf b/masq.tf index 36ce6e21a8..a78d263cef 100644 --- a/masq.tf +++ b/masq.tf @@ -41,4 +41,4 @@ EOF } depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] -} \ No newline at end of file +} diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md new file mode 100644 index 0000000000..0fbcb1ffac --- /dev/null +++ b/modules/private-cluster/README.md @@ -0,0 +1,322 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. This particular submodule creates a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters) +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `network_policy` is true + +## Usage +There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: + +```hcl +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + http_load_balancing = false + horizontal_pod_autoscaling = true + kubernetes_dashboard = true + network_policy = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "10.0.0.0/28" + + node_pools = [ + { + name = "default-node-pool" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 100 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + }, + ] + + node_pools_labels = { + all = {} + + default-node-pool = { + default-node-pool = "true" + } + } + + node_pools_metadata = { + all = {} + + default-node-pool = { + node-pool-metadata-custom-value = "my-node-pool" + } + } + + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = "true" + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + + node_pools_tags = { + all = [] + + default-node-pool = [ + "default-node-pool", + ] + } +} +``` + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| description | The description of the cluster | string | `` | no | +| enable_private_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `false` | no | +| enable_private_nodes | (Beta) Whether nodes have internal IP addresses only | string | `false` | no | +| horizontal_pod_autoscaling | Enable horizontal pod autoscaling addon | string | `true` | no | +| http_load_balancing | Enable httpload balancer addon | string | `true` | no | +| ip_masq_link_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `false` | no | +| ip_masq_resync_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `60s` | no | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| kubernetes_dashboard | Enable kubernetes dashboard addon | string | `false` | no | +| kubernetes_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `latest` | no | +| logging_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `logging.googleapis.com` | no | +| maintenance_start_time | Time window specified for daily maintenance operations in RFC3339 format | string | `05:00` | no | +| master_authorized_networks_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| master_ipv4_cidr_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `10.0.0.0/28` | no | +| monitoring_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `monitoring.googleapis.com` | no | +| name | The name of the cluster (required) | string | - | yes | +| network | The VPC network to host the cluster in (required) | string | - | yes | +| network_policy | Enable network policy addon | string | `false` | no | +| network_project_id | The project ID of the shared VPC's host (for shared vpc support) | string | `` | no | +| node_pools | List of maps containing node pools | list | `` | no | +| node_pools_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node_pools_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node_pools_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | +| non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project_id | The project ID to host the cluster in (required) | string | - | yes | +| region | The region to host the cluster in (required) | string | - | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | +| remove_default_node_pool | Remove default node pool while setting up the cluster | string | `false` | no | +| service_account | The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account | string | `` | no | +| stub_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | - | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal_pod_autoscaling_enabled | Whether horizontal pod autoscaling enabled | +| http_load_balancing_enabled | Whether http load balancing enabled | +| kubernetes_dashboard_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging_service | Logging service used | +| master_authorized_networks_config | Networks from which access to master is permitted | +| master_version | Current master kubernetes version | +| min_master_version | Minimum master kubernetes version | +| monitoring_service | Monitoring service used | +| name | Cluster name | +| network_policy_enabled | Whether network policy enabled | +| node_pools_names | List of node pools names | +| node_pools_versions | List of node pools versions | +| region | Cluster region | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [terraform-provider-google-beta](https://github.com/terraform-providers/terraform-provider-google-beta) v1.20.0 + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountUser + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +## File structure +The project has the following folders and files: + +- /: root folder +- /examples: examples for using this module +- /helpers: Helper scripts +- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file) +- /test: Folders with files for testing the module (see Testing section on this file) +- /main.tf: main file for this module, contains all the resources to create +- /variables.tf: all the variables for the module +- /output.tf: the outputs of the module +- /readme.MD: this file + +## Templating + +To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](/autogen) directory. + +The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate. + +## Testing + +### Requirements +- [bundler](https://github.com/bundler/bundler) +- [gcloud](https://cloud.google.com/sdk/install) +- [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Integration test + +Integration tests are run though [test-kitchen](https://github.com/test-kitchen/test-kitchen), [kitchen-terraform](https://github.com/newcontext-oss/kitchen-terraform), and [InSpec](https://github.com/inspec/inspec). + +Six test-kitchen instances are defined: + +- `deploy-service` +- `node-pool` +- `shared-vpc` +- `simple-regional` +- `simple-zonal` +- `stub-domains` + +The test-kitchen instances in `test/fixtures/` wrap identically-named examples in the `examples/` directory. + +#### Setup + +1. Configure the [test fixtures](#test-configuration) +2. Download a Service Account key with the necessary permissions and put it in the module's root directory with the name `credentials.json`. +3. Build the Docker container for testing: + + ``` + make docker_build_kitchen_terraform + ``` +4. Run the testing container in interactive mode: + + ``` + make docker_run + ``` + + The module root directory will be loaded into the Docker container at `/cft/workdir/`. +5. Run kitchen-terraform to test the infrastructure: + + 1. `kitchen create` creates Terraform state and downloads modules, if applicable. + 2. `kitchen converge` creates the underlying resources. Run `kitchen converge ` to create resources for a specific test case. + 3. Run `kitchen converge` again. This is necessary due to an oddity in how `networkPolicyConfig` is handled by the upstream API. (See [#72](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/72) for details). + 4. `kitchen verify` tests the created infrastructure. Run `kitchen verify ` to run a specific test case. + 5. `kitchen destroy` tears down the underlying resources created by `kitchen converge`. Run `kitchen destroy ` to tear down resources for a specific test case. + +Alternatively, you can simply run `make test_integration_docker` to run all the test steps non-interactively. + +#### Test configuration + +Each test-kitchen instance is configured with a `variables.tfvars` file in the test fixture directory, e.g. `test/fixtures/node_pool/terraform.tfvars`. +For convenience, since all of the variables are project-specific, these files have been symlinked to `test/fixtures/shared/terraform.tfvars`. +Similarly, each test fixture has a `variables.tf` to define these variables, and an `outputs.tf` to facilitate providing necessary information for `inspec` to locate and query against created resources. + +Each test-kitchen instance creates a GCP Network and Subnetwork fixture to house resources, and may create any other necessary fixture data as needed. + +### Autogeneration of documentation from .tf files +Run +``` +make generate_docs +``` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running go fmt and go vet +Running terraform validate +Running hadolint on Dockerfiles +Checking for required files +Testing the validity of the header check +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.026s + +OK +Checking file headers +The following lines have trailing whitespace +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Python - flake8. Can be installed with 'pip install flake8' +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew diff --git a/modules/private-cluster/auth.tf b/modules/private-cluster/auth.tf new file mode 100644 index 0000000000..0bbafaf4a2 --- /dev/null +++ b/modules/private-cluster/auth.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Retrieve authentication token + *****************************************/ +data "google_client_config" "default" { + provider = "google-beta" +} + +/****************************************** + Configure provider + *****************************************/ +provider "kubernetes" { + load_config_file = false + host = "https://${local.cluster_endpoint}" + token = "${data.google_client_config.default.access_token}" + cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" +} diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf new file mode 100644 index 0000000000..947682a292 --- /dev/null +++ b/modules/private-cluster/cluster_regional.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create regional cluster + *****************************************/ +resource "google_container_cluster" "primary" { + provider = "google-beta" + count = "${var.regional ? 1 : 0}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + region = "${var.region}" + additional_zones = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", var.service_account)}" + } + } + + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create regional node pools + *****************************************/ +resource "google_container_node_pool" "pools" { + provider = "google-beta" + count = "${var.regional ? length(var.node_pools) : 0}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + region = "${var.region}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", var.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.primary"] +} + +resource "null_resource" "wait_for_regional_cluster" { + count = "${var.regional ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] +} diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf new file mode 100644 index 0000000000..41dd02f926 --- /dev/null +++ b/modules/private-cluster/cluster_zonal.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Create zonal cluster + *****************************************/ +resource "google_container_cluster" "zonal_primary" { + provider = "google-beta" + count = "${var.regional ? 0 : 1}" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + zone = "${var.zones[0]}" + additional_zones = ["${slice(var.zones,1,length(var.zones))}"] + + network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" + subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" + min_master_version = "${local.kubernetes_version}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", var.service_account)}" + } + } + + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + + remove_default_node_pool = "${var.remove_default_node_pool}" +} + +/****************************************** + Create zonal node pools + *****************************************/ +resource "google_container_node_pool" "zonal_pools" { + provider = "google-beta" + count = "${var.regional ? 0 : length(var.node_pools)}" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + zone = "${var.zones[0]}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")])}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", var.service_account)}" + preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.zonal_primary"] +} + +resource "null_resource" "wait_for_zonal_cluster" { + count = "${var.regional ? 0 : 1}" + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" + } + + depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf new file mode 100644 index 0000000000..25effe580a --- /dev/null +++ b/modules/private-cluster/dns.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +/****************************************** + Delete default kube-dns configmap + *****************************************/ +resource "null_resource" "delete_default_kube_dns_configmap" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + provisioner "local-exec" { + command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" + } + + depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] +} + +/****************************************** + Create kube-dns confimap + *****************************************/ +resource "kubernetes_config_map" "kube-dns" { + count = "${local.custom_kube_dns_config ? 1 : 0}" + + metadata { + name = "kube-dns" + namespace = "kube-system" + + labels { + maintained_by = "terraform" + } + } + + data { + stubDomains = <&2 echo "3 arguments expected. Exiting." + exit 1 +fi + +RESOURCE_NAMESPACE=$1 +RESOURCE_TYPE=$2 +RESOURCE_NAME=$3 + +RESOURCE_LIST=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" || exit 1) + +# Delete requested resource +if [[ $RESOURCE_LIST = *"${RESOURCE_NAME}"* ]]; then + RESOURCE_MAINTAINED_LABEL=$(kubectl -n "${RESOURCE_NAMESPACE}" get "${RESOURCE_TYPE}" -o json "${RESOURCE_NAME}" | jq -r '.metadata.labels."maintained_by"') + if [[ $RESOURCE_MAINTAINED_LABEL = "terraform" ]]; then + echo "Terraform maintained ${RESOURCE_NAME} ${RESOURCE_TYPE} appears to have already been created in ${RESOURCE_NAMESPACE} namespace" + else + echo "Deleting default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" + kubectl -n "${RESOURCE_NAMESPACE}" delete "${RESOURCE_TYPE}" "${RESOURCE_NAME}" + fi +else + echo "No default ${RESOURCE_NAME} ${RESOURCE_TYPE} found in ${RESOURCE_NAMESPACE} namespace" +fi \ No newline at end of file diff --git a/modules/private-cluster/scripts/kubectl_wrapper.sh b/modules/private-cluster/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..255c128223 --- /dev/null +++ b/modules/private-cluster/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" \ No newline at end of file diff --git a/modules/private-cluster/scripts/wait-for-cluster.sh b/modules/private-cluster/scripts/wait-for-cluster.sh new file mode 100755 index 0000000000..3ea1dd78ed --- /dev/null +++ b/modules/private-cluster/scripts/wait-for-cluster.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [[ "${current_status}" == "RECONCILING" ]]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" \ No newline at end of file diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf new file mode 100644 index 0000000000..eabed2c2a6 --- /dev/null +++ b/modules/private-cluster/variables.tf @@ -0,0 +1,228 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + description = "The name of the cluster (required)" +} + +variable "description" { + description = "The description of the cluster" + default = "" +} + +variable "regional" { + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = "list" + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [""] +} + +variable "network" { + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} + +variable "node_version" { + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = "list" + + description = </dev/null kubectl config use-context kubectl-wrapper 1>/dev/null kubectl version 1>/dev/null -"$@" +"$@" \ No newline at end of file diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh index 6ff3253d58..3ea1dd78ed 100755 --- a/scripts/wait-for-cluster.sh +++ b/scripts/wait-for-cluster.sh @@ -30,4 +30,4 @@ while [[ "${current_status}" == "RECONCILING" ]]; do current_status=$($gcloud_command | jq -r "$jq_query") done -echo "Cluster is ready!" +echo "Cluster is ready!" \ No newline at end of file diff --git a/test/fixtures/simple_regional_private/example.tf b/test/fixtures/simple_regional_private/example.tf new file mode 100644 index 0000000000..ddd8058572 --- /dev/null +++ b/test/fixtures/simple_regional_private/example.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_regional_private" + + project_id = "${var.project_id}" + credentials_path = "${local.credentials_path}" + cluster_name_suffix = "-${random_string.suffix.result}" + region = "${var.region}" + network = "${google_compute_network.main.name}" + subnetwork = "${google_compute_subnetwork.main.name}" + ip_range_pods = "${google_compute_subnetwork.main.secondary_ip_range.0.range_name}" + ip_range_services = "${google_compute_subnetwork.main.secondary_ip_range.1.range_name}" + compute_engine_service_account = "${var.compute_engine_service_account}" +} diff --git a/test/fixtures/simple_regional_private/network.tf b/test/fixtures/simple_regional_private/network.tf new file mode 100644 index 0000000000..6228192e31 --- /dev/null +++ b/test/fixtures/simple_regional_private/network.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + credentials_path = "${path.module}/${var.credentials_path_relative}" +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google-beta" { + credentials = "${file(local.credentials_path)}" + project = "${var.project_id}" +} + +resource "google_compute_network" "main" { + project = "${var.project_id}" + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = "false" +} + +resource "google_compute_subnetwork" "main" { + project = "${var.project_id}" + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "${var.region}" + network = "${google_compute_network.main.self_link}" + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} diff --git a/test/fixtures/simple_regional_private/outputs.tf b/test/fixtures/simple_regional_private/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/simple_regional_private/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/simple_regional_private/terraform.tfvars b/test/fixtures/simple_regional_private/terraform.tfvars new file mode 120000 index 0000000000..08ac6f4724 --- /dev/null +++ b/test/fixtures/simple_regional_private/terraform.tfvars @@ -0,0 +1 @@ +../shared/terraform.tfvars \ No newline at end of file diff --git a/test/fixtures/simple_regional_private/variables.tf b/test/fixtures/simple_regional_private/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/simple_regional_private/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/fixtures/simple_zonal_private/example.tf b/test/fixtures/simple_zonal_private/example.tf new file mode 100644 index 0000000000..b45355ccf9 --- /dev/null +++ b/test/fixtures/simple_zonal_private/example.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_zonal_private" + + project_id = "${var.project_id}" + credentials_path = "${local.credentials_path}" + cluster_name_suffix = "-${random_string.suffix.result}" + region = "${var.region}" + zones = ["${slice(var.zones,0,1)}"] + network = "${google_compute_network.main.name}" + subnetwork = "${google_compute_subnetwork.main.name}" + ip_range_pods = "${google_compute_subnetwork.main.secondary_ip_range.0.range_name}" + ip_range_services = "${google_compute_subnetwork.main.secondary_ip_range.1.range_name}" + compute_engine_service_account = "${var.compute_engine_service_account}" +} diff --git a/test/fixtures/simple_zonal_private/network.tf b/test/fixtures/simple_zonal_private/network.tf new file mode 100644 index 0000000000..6228192e31 --- /dev/null +++ b/test/fixtures/simple_zonal_private/network.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + credentials_path = "${path.module}/${var.credentials_path_relative}" +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google-beta" { + credentials = "${file(local.credentials_path)}" + project = "${var.project_id}" +} + +resource "google_compute_network" "main" { + project = "${var.project_id}" + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = "false" +} + +resource "google_compute_subnetwork" "main" { + project = "${var.project_id}" + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "${var.region}" + network = "${google_compute_network.main.self_link}" + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} diff --git a/test/fixtures/simple_zonal_private/outputs.tf b/test/fixtures/simple_zonal_private/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/simple_zonal_private/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/simple_zonal_private/terraform.tfvars b/test/fixtures/simple_zonal_private/terraform.tfvars new file mode 120000 index 0000000000..08ac6f4724 --- /dev/null +++ b/test/fixtures/simple_zonal_private/terraform.tfvars @@ -0,0 +1 @@ +../shared/terraform.tfvars \ No newline at end of file diff --git a/test/fixtures/simple_zonal_private/variables.tf b/test/fixtures/simple_zonal_private/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/simple_zonal_private/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/simple_regional/controls/gcloud.rb b/test/integration/simple_regional/controls/gcloud.rb index c8a5e61bca..83f33be1c3 100644 --- a/test/integration/simple_regional/controls/gcloud.rb +++ b/test/integration/simple_regional/controls/gcloud.rb @@ -42,6 +42,10 @@ expect(data['location']).to match(/^.*[1-9]$/) end + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + it "has the expected addon settings" do expect(data['addonsConfig']).to eq({ "horizontalPodAutoscaling" => {}, diff --git a/test/integration/simple_regional_private/controls/gcloud.rb b/test/integration/simple_regional_private/controls/gcloud.rb new file mode 100644 index 0000000000..18ee9e0293 --- /dev/null +++ b/test/integration/simple_regional_private/controls/gcloud.rb @@ -0,0 +1,167 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +credentials_path = attribute('credentials_path') +ENV['CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE'] = credentials_path + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/simple_regional_private/inspec.yml b/test/integration/simple_regional_private/inspec.yml new file mode 100644 index 0000000000..7ee916787d --- /dev/null +++ b/test/integration/simple_regional_private/inspec.yml @@ -0,0 +1,20 @@ +name: simple_regional_private +attributes: + - name: project_id + required: true + type: string + - name: credentials_path + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/simple_zonal/controls/gcloud.rb b/test/integration/simple_zonal/controls/gcloud.rb index bd21d8ac58..cfd45420e8 100644 --- a/test/integration/simple_zonal/controls/gcloud.rb +++ b/test/integration/simple_zonal/controls/gcloud.rb @@ -46,6 +46,10 @@ expect(data['locations'].size).to eq 1 end + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + it "has the expected addon settings" do expect(data['addonsConfig']).to eq({ "horizontalPodAutoscaling" => {}, diff --git a/test/integration/simple_zonal_private/controls/gcloud.rb b/test/integration/simple_zonal_private/controls/gcloud.rb new file mode 100644 index 0000000000..a2bc62eba9 --- /dev/null +++ b/test/integration/simple_zonal_private/controls/gcloud.rb @@ -0,0 +1,171 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +credentials_path = attribute('credentials_path') +ENV['CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE'] = credentials_path + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses the private master endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/simple_zonal_private/inspec.yml b/test/integration/simple_zonal_private/inspec.yml new file mode 100644 index 0000000000..7ee916787d --- /dev/null +++ b/test/integration/simple_zonal_private/inspec.yml @@ -0,0 +1,20 @@ +name: simple_regional_private +attributes: + - name: project_id + required: true + type: string + - name: credentials_path + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/variables.tf b/variables.tf index ee41bb552c..b185c9b7d5 100644 --- a/variables.tf +++ b/variables.tf @@ -210,4 +210,4 @@ variable "monitoring_service" { variable "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`. Defaults to the compute engine default service account" default = "" -} \ No newline at end of file +}