From b53aaca512b43860854756a0796327124f826470 Mon Sep 17 00:00:00 2001 From: Debjani Banerjee Date: Thu, 9 Mar 2023 22:53:07 +0000 Subject: [PATCH] Adding IAM support for Dataplex Lake, Zone and Asset --- mmv1/api/resource/iam_policy.rb | 5 ++ mmv1/products/dataplex/api.yaml | 67 +++++++++++++++++++ mmv1/products/dataplex/terraform.yaml | 67 +++++++++++++++++++ .../base_configs/iam_test_file.go.erb | 4 +- .../examples/dataplex_asset_primary.tf.erb | 61 +++++++++++++++++ .../examples/dataplex_lake_primary.tf.erb | 13 ++++ .../examples/dataplex_zone_primary.tf.erb | 34 ++++++++++ .../overrides/dataplex/tpgtools_product.yaml | 6 ++ 8 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 mmv1/products/dataplex/api.yaml create mode 100644 mmv1/products/dataplex/terraform.yaml create mode 100644 mmv1/templates/terraform/examples/dataplex_asset_primary.tf.erb create mode 100644 mmv1/templates/terraform/examples/dataplex_lake_primary.tf.erb create mode 100644 mmv1/templates/terraform/examples/dataplex_zone_primary.tf.erb create mode 100644 tpgtools/overrides/dataplex/tpgtools_product.yaml diff --git a/mmv1/api/resource/iam_policy.rb b/mmv1/api/resource/iam_policy.rb index 4cd4204b7084..b1669c4222a0 100644 --- a/mmv1/api/resource/iam_policy.rb +++ b/mmv1/api/resource/iam_policy.rb @@ -110,6 +110,10 @@ class IamPolicy < Api::Object # If unset, defaults to 'ga' attr_reader :min_version + # [Optional] Check to see if zone value should be replaced with GOOGLE_ZONE in iam tests + # Defaults to true + attr_reader :substitute_zone_value + def validate super @@ -138,6 +142,7 @@ def validate ) check :iam_policy_version, type: String check :min_version, type: String + check :substitute_zone_value, type: :boolean, default: true end end end diff --git a/mmv1/products/dataplex/api.yaml b/mmv1/products/dataplex/api.yaml new file mode 100644 index 000000000000..3312cc8f7834 --- /dev/null +++ b/mmv1/products/dataplex/api.yaml @@ -0,0 +1,67 @@ +# Copyright 2020 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +!ruby/object:Api::Product +name: Dataplex +display_name: Cloud Dataplex +versions: + - !ruby/object:Api::Product::Version + name: ga + base_url: https://dataplex.googleapis.com/v1/ +scopes: + - https://www.googleapis.com/auth/cloud-platform +apis_required: + - !ruby/object:Api::Product::ApiReference + name: Cloud Dataplex API + url: https://cloud.google.com/dataplex/docs/reference/rest/ +objects: + - !ruby/object:Api::Resource + name: "Lake" + base_url: "projects/{{project}}/locations/{{location}}/lakes/{{name}}" + self_link: "projects/{{project}}/locations/{{location}}/lakes/{{name}}" + exclude_resource: true + description: | + Only used to generate IAM resources + properties: + - !ruby/object:Api::Type::String + name: "name" + output: true + description: | + Dummy property. + - !ruby/object:Api::Resource + name: "Zone" + base_url: "projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{name}}" + self_link: "projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{name}}" + exclude_resource: true + description: | + Only used to generate IAM resources + properties: + - !ruby/object:Api::Type::String + name: "name" + output: true + description: | + Dummy property. + - !ruby/object:Api::Resource + name: "Asset" + base_url: "projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{dataplex_zone}}/assets/{{name}}" + self_link: "projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{dataplex_zone}}/assets/{{name}}" + exclude_resource: true + description: | + Only used to generate IAM resources + properties: + - !ruby/object:Api::Type::String + name: "name" + output: true + description: | + Dummy property diff --git a/mmv1/products/dataplex/terraform.yaml b/mmv1/products/dataplex/terraform.yaml new file mode 100644 index 000000000000..0f127bf7b1a2 --- /dev/null +++ b/mmv1/products/dataplex/terraform.yaml @@ -0,0 +1,67 @@ +# Copyright 2020 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +!ruby/object:Provider::Terraform::Config +overrides: !ruby/object:Overrides::ResourceOverrides + Lake: !ruby/object:Overrides::Terraform::ResourceOverride + iam_policy: !ruby/object:Api::Resource::IamPolicy + exclude: false + method_name_separator: ":" + parent_resource_attribute: "lake" + fetch_iam_policy_verb: :GET + import_format: ["projects/{{project}}/locations/{{location}}/lakes/{{name}}", "{{name}}"] + examples: + - !ruby/object:Provider::Terraform::Examples + name: "dataplex_lake_primary" + primary_resource_id: "example" + primary_resource_name: "fmt.Sprintf(\"tf-test-lake%s\", context[\"random_suffix\"])" + test_env_vars: + project_name: :PROJECT_NAME + Zone: !ruby/object:Overrides::Terraform::ResourceOverride + iam_policy: !ruby/object:Api::Resource::IamPolicy + exclude: false + substitute_zone_value: false + method_name_separator: ":" + parent_resource_attribute: "dataplex_zone" + fetch_iam_policy_verb: :GET + import_format: ["projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{name}}", "{{name}}"] + examples: + - !ruby/object:Provider::Terraform::Examples + name: "dataplex_zone_primary" + primary_resource_id: "example" + primary_resource_name: "fmt.Sprintf(\"tf-test-lake%s\", context[\"random_suffix\"]), fmt.Sprintf(\"tf-test-zone%s\", context[\"random_suffix\"])" + test_env_vars: + project_name: :PROJECT_NAME + Asset: !ruby/object:Overrides::Terraform::ResourceOverride + iam_policy: !ruby/object:Api::Resource::IamPolicy + exclude: false + method_name_separator: ":" + parent_resource_attribute: "asset" + fetch_iam_policy_verb: :GET + import_format: ["projects/{{project}}/locations/{{location}}/lakes/{{lake}}/zones/{{dataplex_zone}}/assets/{{name}}", "{{name}}"] + examples: + - !ruby/object:Provider::Terraform::Examples + name: "dataplex_asset_primary" + primary_resource_id: "example" + primary_resource_name: "fmt.Sprintf(\"tf-test-lake%s\", context[\"random_suffix\"]), fmt.Sprintf(\"tf-test-zone%s\", context[\"random_suffix\"]), fmt.Sprintf(\"tf-test-asset%s\", context[\"random_suffix\"])" + test_env_vars: + project_name: :PROJECT_NAME +# This is for copying files over +files: !ruby/object:Provider::Config::Files + # These files have templating (ERB) code that will be run. + # This is usually to add licensing info, autogeneration notices, etc. + compile: +<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%> + diff --git a/mmv1/templates/terraform/examples/base_configs/iam_test_file.go.erb b/mmv1/templates/terraform/examples/base_configs/iam_test_file.go.erb index c8bf56ae90c4..b875b03402b4 100644 --- a/mmv1/templates/terraform/examples/base_configs/iam_test_file.go.erb +++ b/mmv1/templates/terraform/examples/base_configs/iam_test_file.go.erb @@ -42,8 +42,8 @@ import_url = import_format.gsub(/({{)%?(\w+)(}})/, '%s').gsub(object.__product.b <% import_qualifiers.push('context["project_id"]') -%> <% end -%> <% end -%> - <% elsif param == 'zone' -%> - <% import_qualifiers.push('GetTestZoneFromEnv()') -%> + <% elsif param == 'zone' and object.iam_policy.substitute_zone_value -%> + <% import_qualifiers.push('GetTestZoneFromEnv()') -%> <% elsif param == 'region' || param == 'location' -%> <% if example.region_override.nil? -%> <% import_qualifiers.push('GetTestRegionFromEnv()') -%> diff --git a/mmv1/templates/terraform/examples/dataplex_asset_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_asset_primary.tf.erb new file mode 100644 index 000000000000..032672dfa89e --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_asset_primary.tf.erb @@ -0,0 +1,61 @@ +resource "google_storage_bucket" "primary_bucket" { + name = "dataplex-bucket-%{random_suffix}" + location = "us-central1" + uniform_bucket_level_access = true + lifecycle { + ignore_changes = [ + labels + ] + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-lake%{random_suffix}" + location = "us-central1" + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + +resource "google_dataplex_zone" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-zone%{random_suffix}" + location = "us-central1" + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + type = "RAW" + + discovery_spec { + enabled = false + } + + + resource_spec { + location_type = "SINGLE_REGION" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + + +resource "google_dataplex_asset" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-asset%{random_suffix}" + location = "us-central1" + + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + dataplex_zone = google_dataplex_zone.<%= ctx[:primary_resource_id] %>.name + discovery_spec { + enabled = false + } + + resource_spec { + name = "projects/<%= ctx[:test_env_vars]['project_name'] %>/buckets/dataplex-bucket-%{random_suffix}" + type = "STORAGE_BUCKET" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" + depends_on = [ + google_storage_bucket.primary_bucket + ] +} + diff --git a/mmv1/templates/terraform/examples/dataplex_lake_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_lake_primary.tf.erb new file mode 100644 index 000000000000..46651fe5145b --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_lake_primary.tf.erb @@ -0,0 +1,13 @@ +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + location = "us-central1" + name = "tf-test-lake%{random_suffix}" + description = "Test Lake" + display_name = "Test Lake" + + labels = { + my-lake = "exists" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + diff --git a/mmv1/templates/terraform/examples/dataplex_zone_primary.tf.erb b/mmv1/templates/terraform/examples/dataplex_zone_primary.tf.erb new file mode 100644 index 000000000000..b5ee3a9404de --- /dev/null +++ b/mmv1/templates/terraform/examples/dataplex_zone_primary.tf.erb @@ -0,0 +1,34 @@ +resource "google_dataplex_lake" "<%= ctx[:primary_resource_id] %>" { + location = "us-central1" + name = "tf-test-lake%{random_suffix}" + description = "Test Lake" + display_name = "Test Lake" + + labels = { + my-lake = "exists" + } + + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + + +resource "google_dataplex_zone" "<%= ctx[:primary_resource_id] %>" { + name = "tf-test-zone%{random_suffix}" + discovery_spec { + enabled = false + } + + lake = google_dataplex_lake.<%= ctx[:primary_resource_id] %>.name + location = "us-central1" + + resource_spec { + location_type = "MULTI_REGION" + } + + type = "RAW" + description = "Test Zone" + display_name = "Test Zone" + labels = {} + project = "<%= ctx[:test_env_vars]['project_name'] %>" +} + diff --git a/tpgtools/overrides/dataplex/tpgtools_product.yaml b/tpgtools/overrides/dataplex/tpgtools_product.yaml new file mode 100644 index 000000000000..75271c791a0c --- /dev/null +++ b/tpgtools/overrides/dataplex/tpgtools_product.yaml @@ -0,0 +1,6 @@ +## product level overrides + +- type: PRODUCT_BASE_PATH + details: + skip: true +