From 785b7042c290d73afe2c5ffffe5f215fc6e1119a Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Thu, 17 Nov 2022 09:53:07 -0800 Subject: [PATCH 01/13] Added accelerators field to Data Fusion --- mmv1/products/datafusion/api.yaml | 26 +++++++++++++++++++ .../examples/data_fusion_instance_full.tf.erb | 7 ++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mmv1/products/datafusion/api.yaml b/mmv1/products/datafusion/api.yaml index 61f33711d8c0..7c6426578cfd 100644 --- a/mmv1/products/datafusion/api.yaml +++ b/mmv1/products/datafusion/api.yaml @@ -161,6 +161,7 @@ objects: Endpoint on which the Data Fusion UI and REST APIs are accessible. - !ruby/object:Api::Type::String name: 'version' + input: true description: | Current version of the Data Fusion. - !ruby/object:Api::Type::String @@ -261,3 +262,28 @@ objects: The resource name of the Pub/Sub topic. Format: projects/{projectId}/topics/{topic_id} required: true input: true + - !ruby/object:Api::Type::Array + name: 'accelerators' + description: | + List of accelerators enabled for this CDF instance. + + If accelerators are enabled it is possible a permadiff will be created with the Options field. + Users will need to either manually update their state file to include these diffed options, or include the field in a [lifecycle ignore changes block](https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes). + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::Enum + name: 'acceleratorType' + description: | + The type of an accelator for a CDF instance. + values: + - :CDC + - :HEALTHCARE + - :CCAI_INSIGHTS + required: true + - !ruby/object:Api::Type::Enum + name: 'state' + description: | + The type of an accelator for a CDF instance. + values: + - :ENABLED + - :DISABLED diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index 44daa5eb109e..d92ccd26f7cc 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -17,8 +17,13 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { network = "default" ip_allocation = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" } - <%= ctx[:vars]['prober_test_run'] %> + version = "6.3.0" + dataproc_service_account = data.google_app_engine_default_service_account.default.email + accelerators { + acceleratorType = "ACCELERATOR_TYPE_UNSPECIFIED" + state = "DISABLED" + } } data "google_app_engine_default_service_account" "default" { From 22e84e5bd4269d12b044dfaaecc80dd0be6ab8d4 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Thu, 17 Nov 2022 15:37:04 -0800 Subject: [PATCH 02/13] adding test update --- .../terraform/examples/data_fusion_instance_full.tf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index d92ccd26f7cc..a544569aac2a 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -18,7 +18,7 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { ip_allocation = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" } <%= ctx[:vars]['prober_test_run'] %> - version = "6.3.0" + version = "6.7.0" dataproc_service_account = data.google_app_engine_default_service_account.default.email accelerators { acceleratorType = "ACCELERATOR_TYPE_UNSPECIFIED" From 2a46afbc2234f186a55d63b3189c0a1ee8fd9d63 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Fri, 18 Nov 2022 15:11:48 -0800 Subject: [PATCH 03/13] Updates made to accelerators tests --- .../terraform/examples/data_fusion_instance_full.tf.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index a544569aac2a..f200627e4592 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -19,6 +19,10 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { } <%= ctx[:vars]['prober_test_run'] %> version = "6.7.0" + accelerators { + accelerator_type = "ACCELERATOR_TYPE_UNSPECIFIED" + state = "STATE_UNSPECIFIED" + } dataproc_service_account = data.google_app_engine_default_service_account.default.email accelerators { acceleratorType = "ACCELERATOR_TYPE_UNSPECIFIED" From 230a82410814b9ec60780f04f4aa829fa534f35e Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Wed, 7 Dec 2022 10:51:30 -0800 Subject: [PATCH 04/13] updating declared accelerator state in example test --- .../terraform/examples/data_fusion_instance_full.tf.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index f200627e4592..c19395ec0b37 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -21,7 +21,6 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { version = "6.7.0" accelerators { accelerator_type = "ACCELERATOR_TYPE_UNSPECIFIED" - state = "STATE_UNSPECIFIED" } dataproc_service_account = data.google_app_engine_default_service_account.default.email accelerators { From 8ce0cc259ed09f6ed78854def59966f39cd86891 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Wed, 18 Jan 2023 14:21:37 -0800 Subject: [PATCH 05/13] Updating test to refer to proper values --- .../terraform/examples/data_fusion_instance_full.tf.erb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index c19395ec0b37..5fd242a47db5 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -20,11 +20,7 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { <%= ctx[:vars]['prober_test_run'] %> version = "6.7.0" accelerators { - accelerator_type = "ACCELERATOR_TYPE_UNSPECIFIED" - } - dataproc_service_account = data.google_app_engine_default_service_account.default.email - accelerators { - acceleratorType = "ACCELERATOR_TYPE_UNSPECIFIED" + accelerator_type = "CDC" state = "DISABLED" } } From e93f4ae78ef7ca8b3e08a7567ade044650d7c36b Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Fri, 20 Jan 2023 09:58:24 -0800 Subject: [PATCH 06/13] Updated field attributes --- mmv1/products/datafusion/terraform.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmv1/products/datafusion/terraform.yaml b/mmv1/products/datafusion/terraform.yaml index b1e718382ace..5d9017779814 100644 --- a/mmv1/products/datafusion/terraform.yaml +++ b/mmv1/products/datafusion/terraform.yaml @@ -78,7 +78,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides custom_expand: 'templates/terraform/custom_expand/shortname_to_url.go.erb' custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb' zone: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true + default_from_api: true + accelerators.state: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true # This is for copying files over files: !ruby/object:Provider::Config::Files # These files have templating (ERB) code that will be run. From 2663402133a2f81152b8ccab6bc2924bd4f3a77b Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Thu, 26 Jan 2023 13:07:38 -0800 Subject: [PATCH 07/13] Added in-place update handwritten test --- .../tests/resource_data_fusion_instance_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go b/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go index 0f82a29fb088..fc96ff38dce7 100644 --- a/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go +++ b/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go @@ -42,12 +42,20 @@ resource "google_data_fusion_instance" "foobar" { name = "%s" region = "us-central1" type = "BASIC" +<<<<<<< HEAD:mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go # See supported versions here https://cloud.google.com/data-fusion/docs/support/version-support-policy version = "6.7.0" +======= + version = "6.5.0" +>>>>>>> ba9999d00 (Added in-place update handwritten test):mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go.erb # Mark for testing to avoid service networking connection usage that is not cleaned up options = { prober_test_run = "true" } + accelerators { + accelerator_type = "CDC" + state = "DISABLED" + } } `, instanceName) } @@ -66,6 +74,11 @@ resource "google_data_fusion_instance" "foobar" { label2 = "value2" } version = "6.8.0" + + accelerators { + accelerator_type = "CCAI_INSIGHTS" + state = "ENABLED" + } # Mark for testing to avoid service networking connection usage that is not cleaned up options = { prober_test_run = "true" From ff94f23612788ebf14a641049a52e4045d41aec7 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Fri, 27 Jan 2023 10:58:07 -0800 Subject: [PATCH 08/13] Removed unneeded value, updated test for better documentation --- .../terraform/examples/data_fusion_instance_full.tf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb index 5fd242a47db5..7b0875af5a85 100644 --- a/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb +++ b/mmv1/templates/terraform/examples/data_fusion_instance_full.tf.erb @@ -21,7 +21,7 @@ resource "google_data_fusion_instance" "<%= ctx[:primary_resource_id] %>" { version = "6.7.0" accelerators { accelerator_type = "CDC" - state = "DISABLED" + state = "ENABLED" } } From a9b94db73657a99984a0ec53380602c72cf27368 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Mon, 27 Feb 2023 02:13:38 -0800 Subject: [PATCH 09/13] Fixing conflict error --- .../terraform/tests/resource_data_fusion_instance_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go b/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go index fc96ff38dce7..ba06bba43b64 100644 --- a/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go +++ b/mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go @@ -42,12 +42,8 @@ resource "google_data_fusion_instance" "foobar" { name = "%s" region = "us-central1" type = "BASIC" -<<<<<<< HEAD:mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go # See supported versions here https://cloud.google.com/data-fusion/docs/support/version-support-policy version = "6.7.0" -======= - version = "6.5.0" ->>>>>>> ba9999d00 (Added in-place update handwritten test):mmv1/third_party/terraform/tests/resource_data_fusion_instance_test.go.erb # Mark for testing to avoid service networking connection usage that is not cleaned up options = { prober_test_run = "true" From 56edbba92cd4430977a4dd47fbce1316ffdd84ad Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Mon, 27 Feb 2023 02:20:03 -0800 Subject: [PATCH 10/13] Fixing conflict error 2 --- mmv1/products/datafusion/api.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/mmv1/products/datafusion/api.yaml b/mmv1/products/datafusion/api.yaml index 7c6426578cfd..d76afd68ed3d 100644 --- a/mmv1/products/datafusion/api.yaml +++ b/mmv1/products/datafusion/api.yaml @@ -161,7 +161,6 @@ objects: Endpoint on which the Data Fusion UI and REST APIs are accessible. - !ruby/object:Api::Type::String name: 'version' - input: true description: | Current version of the Data Fusion. - !ruby/object:Api::Type::String From b216d4e5e06d18048f35db48dc367e2d9b015d25 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Mon, 6 Mar 2023 01:12:28 -0800 Subject: [PATCH 11/13] Updated to add diffsuppressfunc --- mmv1/products/datafusion/terraform.yaml | 3 +++ .../data_fusion_instance_option.go.erb | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 mmv1/templates/terraform/constants/data_fusion_instance_option.go.erb diff --git a/mmv1/products/datafusion/terraform.yaml b/mmv1/products/datafusion/terraform.yaml index 5d9017779814..73bb3fe63620 100644 --- a/mmv1/products/datafusion/terraform.yaml +++ b/mmv1/products/datafusion/terraform.yaml @@ -77,6 +77,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides name: !ruby/object:Overrides::Terraform::PropertyOverride custom_expand: 'templates/terraform/custom_expand/shortname_to_url.go.erb' custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb' + options: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true + diff_suppress_func: 'instanceOptionsDiffSuppress' zone: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true accelerators.state: !ruby/object:Overrides::Terraform::PropertyOverride diff --git a/mmv1/templates/terraform/constants/data_fusion_instance_option.go.erb b/mmv1/templates/terraform/constants/data_fusion_instance_option.go.erb new file mode 100644 index 000000000000..ede5560480b8 --- /dev/null +++ b/mmv1/templates/terraform/constants/data_fusion_instance_option.go.erb @@ -0,0 +1,21 @@ +var instanceAcceleratorOptions = []string{ + "delta.default.checkpoint.directory", + "ui.feature.cdc", +} + +func instanceOptionsDiffSuppress(k, old, new string, d *schema.ResourceData) bool { + // Suppress diffs for the options generated by adding an accelerator to a data fusion instance + for _, option := range instanceAcceleratorOptions { + if strings.Contains(k, option) && new == "" { + return true + } + } + + // Let diff be determined by options (above) + if strings.Contains(k, "options.%") { + return true + } + + // For other keys, don't suppress diff. + return false +} \ No newline at end of file From 3d6e88078229e2b1c028985909f56ae5b1b578e7 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Mon, 6 Mar 2023 02:23:22 -0800 Subject: [PATCH 12/13] added custom code field to terraform.yaml --- mmv1/products/datafusion/terraform.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/products/datafusion/terraform.yaml b/mmv1/products/datafusion/terraform.yaml index 73bb3fe63620..3cfd257d3413 100644 --- a/mmv1/products/datafusion/terraform.yaml +++ b/mmv1/products/datafusion/terraform.yaml @@ -67,6 +67,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides instance_name: "my-instance" custom_code: !ruby/object:Provider::Terraform::CustomCode pre_update: templates/terraform/pre_update/datafusion_instance_update.go.erb + constants: templates/terraform/constants/data_fusion_instance_option.go.erb properties: region: !ruby/object:Overrides::Terraform::PropertyOverride ignore_read: true From dae9d117b837bc504ce079f8da6a9b6cdf352d2c Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Thu, 9 Mar 2023 12:10:28 -0800 Subject: [PATCH 13/13] Making suggested default behavior functionality updates. --- mmv1/products/datafusion/Instance.yaml | 1 + mmv1/products/datafusion/terraform.yaml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mmv1/products/datafusion/Instance.yaml b/mmv1/products/datafusion/Instance.yaml index 8322fd02b88d..e7cbd9b395b1 100644 --- a/mmv1/products/datafusion/Instance.yaml +++ b/mmv1/products/datafusion/Instance.yaml @@ -252,3 +252,4 @@ properties: values: - :ENABLED - :DISABLED + required: true diff --git a/mmv1/products/datafusion/terraform.yaml b/mmv1/products/datafusion/terraform.yaml index 3cfd257d3413..b250cb6f643c 100644 --- a/mmv1/products/datafusion/terraform.yaml +++ b/mmv1/products/datafusion/terraform.yaml @@ -82,9 +82,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides default_from_api: true diff_suppress_func: 'instanceOptionsDiffSuppress' zone: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true - accelerators.state: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true + default_from_api: true # This is for copying files over files: !ruby/object:Provider::Config::Files # These files have templating (ERB) code that will be run.