chore: Use devtools generate_modules helper (#433)

* Use devtools generate_modules helper.

* [autogen_modules] Add 'template_folder' param, add safer-cluster modules config.

* contrib

* Remove check_generate from this module

* Remove custom check step from lint CI config

* Tweak indentity namespace description

Co-authored-by: Osman Mazinov <53472807+omazin@users.noreply.github.com>

CONTRIBUTING.md

@@ -21,10 +21,10 @@ must be refreshed if the module interfaces are changed.
 
 To more cleanly handle cases where desired functionality would require complex duplication of Terraform resources (i.e. [PR 51](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/51)), this repository is largely generated from the [`autogen`](/autogen) directory.
 
-The root module is generated by running `make generate`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate.
+The root module is generated by running `make build`. Changes to this repository should be made in the [`autogen`](/autogen) directory where appropriate.
 
 Note: The correct sequence to update the repo using autogen functionality is to run
-`make docker_generate && make docker_generate_docs`.  This will create the various Terraform files, and then
+`make build`.  This will create the various Terraform files, and then
 generate the Terraform documentation using `terraform-docs`.
 
 ### Autogeneration of documentation from .tf files

Makefile

@@ -81,16 +81,19 @@ docker_generate_docs:
 		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs'
 
 # Generate files from autogen
-.PHONY: docker_generate
-docker_generate:
+.PHONY: docker_generate_modules
+docker_generate_modules:
 	docker run --rm -it \
                 -v "$(CURDIR)":/workspace \
                 $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
-                /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate'
+                /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_modules'
 
 # Alias for backwards compatibility
 .PHONY: generate_docs
 generate_docs: docker_generate_docs
 
 .PHONY: generate
-generate: docker_generate
+generate: docker_generate_modules
+
+.PHONY: build
+build: docker_generate_modules docker_generate_docs

autogen/main/variables.tf.tmpl

@@ -451,7 +451,7 @@ variable "enable_vertical_pod_autoscaling" {
 }
 
 variable "identity_namespace" {
-  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`)"
+  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)"
   type        = string
   default     = "enabled"
 }

autogen_modules.json

@@ -0,0 +1,69 @@
+[
+  {
+    "template_folder": "./autogen/main",
+    "path": "./",
+    "options": {
+      "private_cluster": false
+    }
+  },
+  {
+    "template_folder": "./autogen/main",
+    "path": "./modules/private-cluster",
+    "options": {
+      "module_path": "//modules/private-cluster",
+      "private_cluster": true
+    }
+  },
+  {
+    "template_folder": "./autogen/main",
+    "path": "./modules/beta-private-cluster",
+    "options": {
+      "module_path": "//modules/beta-private-cluster",
+      "private_cluster": true,
+      "beta_cluster": true
+    }
+  },
+  {
+    "template_folder": "./autogen/main",
+    "path": "./modules/private-cluster-update-variant",
+    "options": {
+      "module_path": "//modules/private-cluster-update-variant",
+      "private_cluster": true,
+      "update_variant": true
+    }
+  },
+  {
+    "template_folder": "./autogen/main",
+    "path": "./modules/beta-private-cluster-update-variant",
+    "options": {
+      "module_path": "//modules/beta-private-cluster-update-variant",
+      "private_cluster": true,
+      "update_variant": true,
+      "beta_cluster": true
+    }
+  },
+  {
+    "template_folder": "./autogen/main",
+    "path": "./modules/beta-public-cluster",
+    "options": {
+      "module_path": "//modules/beta-public-cluster",
+      "private_cluster": false,
+      "beta_cluster": true
+    }
+  },
+  {
+    "template_folder": "./autogen/safer-cluster",
+    "path": "./modules/safer-cluster",
+    "options": {
+      "module_path": "//modules/safer-cluster"
+    }
+  },
+  {
+    "template_folder": "./autogen/safer-cluster",
+    "path": "./modules/safer-cluster-update-variant",
+    "options": {
+      "module_path": "//modules/safer-cluster-update-variant",
+      "update_variant": true
+    }
+  }
+]

build/lint.cloudbuild.yaml

@@ -13,9 +13,6 @@
 # limitations under the License.
 
 steps:
-- id: 'lint-generation'
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && check_generate']
 - id: 'lint-tests'
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/usr/local/bin/test_lint.sh']

modules/beta-private-cluster-update-variant/README.md

@@ -179,7 +179,7 @@ Then perform the following commands on the root folder:
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`) | string | `"enabled"` | no |
+| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |

modules/beta-private-cluster-update-variant/variables.tf

@@ -444,7 +444,7 @@ variable "enable_vertical_pod_autoscaling" {
 }
 
 variable "identity_namespace" {
-  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`)"
+  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)"
   type        = string
   default     = "enabled"
 }

modules/beta-private-cluster/README.md

@@ -157,7 +157,7 @@ Then perform the following commands on the root folder:
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`) | string | `"enabled"` | no |
+| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |

modules/beta-private-cluster/variables.tf

@@ -444,7 +444,7 @@ variable "enable_vertical_pod_autoscaling" {
 }
 
 variable "identity_namespace" {
-  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`)"
+  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)"
   type        = string
   default     = "enabled"
 }

modules/beta-public-cluster/README.md

@@ -136,7 +136,7 @@ Then perform the following commands on the root folder:
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`) | string | `"enabled"` | no |
+| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |

modules/beta-public-cluster/variables.tf

@@ -420,7 +420,7 @@ variable "enable_vertical_pod_autoscaling" {
 }
 
 variable "identity_namespace" {
-  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `project_id..svc.id.goog`)"
+  description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)"
   type        = string
   default     = "enabled"
 }

test/task_helper_functions.sh

@@ -14,43 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-function generate() {
-  pip3 install --user -r /workspace/helpers/generate_modules/requirements.txt
-  /workspace/helpers/generate_modules/generate_modules.py
-}
-
-# Changed from using git-diff, to aviod errors on CI:
-# fatal: not a git repository (or any parent up to mount point /)
-function check_generate() {
-  local tempdir rval rc
-  setup_trap_handler
-  tempdir=$(mktemp -d)
-  rval=0
-  echo "Checking submodule's files generation"
-  rsync -axh \
-    --exclude '*/.terraform' \
-    --exclude '*/.kitchen' \
-    --exclude '*/.git' \
-    /workspace "${tempdir}" >/dev/null 2>/dev/null
-  cd "${tempdir}/workspace" || exit 1
-  generate >/dev/null 2>/dev/null
-  generate_docs >/dev/null 2>/dev/null
-  diff -r \
-    --exclude=".terraform" \
-    --exclude=".kitchen" \
-    --exclude=".git" \
-    /workspace "${tempdir}/workspace"
-  rc=$?
-  if [[ "${rc}" -ne 0 ]]; then
-    echo "Error: submodule's files generation has not been run, please run the"
-    echo "'source /workspace/helpers/generate.sh && generate' commands and commit the above changes."
-    ((rval++))
-  fi
-  cd /workspace || exit 1
-  rm -Rf "${tempdir}"
-  return $((rval))
-}
-
 # Pre-download the Anthos Config Management operator
 function download_acm() {
    gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml /workspace/acm.yaml