From 76f5160a7d0ab299e70146ce8e200d8c084ba489 Mon Sep 17 00:00:00 2001 From: The Magician Date: Mon, 23 Jan 2023 08:38:10 -0800 Subject: [PATCH] Update datastream_stream resource to include kms fields (#7126) (#13549) * Update datastream_stream resource to include kms fields * Add skip_vcr back Signed-off-by: Modular Magician Signed-off-by: Modular Magician --- .changelog/7126.txt | 3 + google/resource_datastream_stream.go | 50 ++++++ ...source_datastream_stream_generated_test.go | 163 ++++++++++++++++++ .../docs/r/datastream_stream.html.markdown | 147 ++++++++++++++++ 4 files changed, 363 insertions(+) create mode 100644 .changelog/7126.txt diff --git a/.changelog/7126.txt b/.changelog/7126.txt new file mode 100644 index 00000000000..515c7327672 --- /dev/null +++ b/.changelog/7126.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +datastream: add `customer_managed_encryption_key` and `destination_config.bigquery_destination_config.source_hierarchy_datasets.dataset_template.kms_key_name` fields to `datastream_stream` resource +``` diff --git a/google/resource_datastream_stream.go b/google/resource_datastream_stream.go index 6fbc77d11f3..64682d8a878 100644 --- a/google/resource_datastream_stream.go +++ b/google/resource_datastream_stream.go @@ -169,6 +169,15 @@ See https://cloud.google.com/bigquery/docs/locations for supported locations.`, Description: `If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _.`, }, + "kms_key_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `Describes the Cloud KMS encryption key that will be used to protect destination BigQuery +table. The BigQuery Service Account associated with your project requires access to this +encryption key. i.e. projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}. +See https://cloud.google.com/bigquery/docs/customer-managed-encryption for more information.`, + }, }, }, }, @@ -576,6 +585,13 @@ https://dev.mysql.com/doc/refman/8.0/en/data-types.html`, }, ExactlyOneOf: []string{"backfill_all", "backfill_none"}, }, + "customer_managed_encryption_key": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data +will be encrypted using an internal Stream-specific encryption key provisioned through KMS.`, + }, "labels": { Type: schema.TypeMap, Optional: true, @@ -653,6 +669,12 @@ func resourceDatastreamStreamCreate(d *schema.ResourceData, meta interface{}) er } else if v, ok := d.GetOkExists("backfill_none"); ok || !reflect.DeepEqual(v, backfillNoneProp) { obj["backfillNone"] = backfillNoneProp } + customerManagedEncryptionKeyProp, err := expandDatastreamStreamCustomerManagedEncryptionKey(d.Get("customer_managed_encryption_key"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("customer_managed_encryption_key"); !isEmptyValue(reflect.ValueOf(customerManagedEncryptionKeyProp)) && (ok || !reflect.DeepEqual(v, customerManagedEncryptionKeyProp)) { + obj["customerManagedEncryptionKey"] = customerManagedEncryptionKeyProp + } obj, err = resourceDatastreamStreamEncoder(d, meta, obj) if err != nil { @@ -787,6 +809,9 @@ func resourceDatastreamStreamRead(d *schema.ResourceData, meta interface{}) erro if err := d.Set("backfill_none", flattenDatastreamStreamBackfillNone(res["backfillNone"], d, config)); err != nil { return fmt.Errorf("Error reading Stream: %s", err) } + if err := d.Set("customer_managed_encryption_key", flattenDatastreamStreamCustomerManagedEncryptionKey(res["customerManagedEncryptionKey"], d, config)); err != nil { + return fmt.Errorf("Error reading Stream: %s", err) + } return nil } @@ -1501,6 +1526,8 @@ func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHier flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateLocation(original["location"], d, config) transformed["dataset_id_prefix"] = flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateDatasetIdPrefix(original["datasetIdPrefix"], d, config) + transformed["kms_key_name"] = + flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateKmsKeyName(original["kmsKeyName"], d, config) return []interface{}{transformed} } func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateLocation(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -1511,6 +1538,10 @@ func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHier return v } +func flattenDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateKmsKeyName(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func flattenDatastreamStreamState(v interface{}, d *schema.ResourceData, config *Config) interface{} { return v } @@ -1670,6 +1701,10 @@ func flattenDatastreamStreamBackfillNone(v interface{}, d *schema.ResourceData, return []interface{}{transformed} } +func flattenDatastreamStreamCustomerManagedEncryptionKey(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func expandDatastreamStreamLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) { if v == nil { return map[string]string{}, nil @@ -2358,6 +2393,13 @@ func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHiera transformed["datasetIdPrefix"] = transformedDatasetIdPrefix } + transformedKmsKeyName, err := expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateKmsKeyName(original["kms_key_name"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedKmsKeyName); val.IsValid() && !isEmptyValue(val) { + transformed["kmsKeyName"] = transformedKmsKeyName + } + return transformed, nil } @@ -2369,6 +2411,10 @@ func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHiera return v, nil } +func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasetsDatasetTemplateKmsKeyName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandDatastreamStreamBackfillAll(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { l := v.([]interface{}) if len(l) == 0 { @@ -2585,6 +2631,10 @@ func expandDatastreamStreamBackfillNone(v interface{}, d TerraformResourceData, return transformed, nil } +func expandDatastreamStreamCustomerManagedEncryptionKey(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func resourceDatastreamStreamEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) { if d.HasChange("desired_state") { obj["state"] = d.Get("desired_state") diff --git a/google/resource_datastream_stream_generated_test.go b/google/resource_datastream_stream_generated_test.go index bae2c07261d..ed13b15d6dc 100644 --- a/google/resource_datastream_stream_generated_test.go +++ b/google/resource_datastream_stream_generated_test.go @@ -189,6 +189,7 @@ func TestAccDatastreamStream_datastreamStreamFullExample(t *testing.T) { context := map[string]interface{}{ "deletion_protection": false, + "stream_cmek": BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name, "random_suffix": randString(t, 10), } @@ -311,6 +312,12 @@ resource "google_storage_bucket_iam_member" "reader" { member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" } +resource "google_kms_crypto_key_iam_member" "key_user" { + crypto_key_id = "%{stream_cmek}" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + resource "google_datastream_connection_profile" "destination_connection_profile" { display_name = "Connection profile" location = "us-central1" @@ -323,6 +330,9 @@ resource "google_datastream_connection_profile" "destination_connection_profile" } resource "google_datastream_stream" "default" { + depends_on = [ + google_kms_crypto_key_iam_member.key_user + ] stream_id = "tf-test-my-stream%{random_suffix}" desired_state = "NOT_STARTED" location = "us-central1" @@ -399,6 +409,159 @@ resource "google_datastream_stream" "default" { } } } + + customer_managed_encryption_key = "%{stream_cmek}" +} +`, context) +} + +func TestAccDatastreamStream_datastreamStreamBigqueryExample(t *testing.T) { + skipIfVcr(t) + t.Parallel() + + context := map[string]interface{}{ + "deletion_protection": false, + "bigquery_destination_table_kms_key_name": BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name, + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + ExternalProviders: map[string]resource.ExternalProvider{ + "random": {}, + "time": {}, + }, + CheckDestroy: testAccCheckDatastreamStreamDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDatastreamStream_datastreamStreamBigqueryExample(context), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + }, + }) +} + +func testAccDatastreamStream_datastreamStreamBigqueryExample(context map[string]interface{}) string { + return Nprintf(` +data "google_project" "project" { +} + +resource "google_sql_database_instance" "instance" { + name = "tf-test-my-instance%{random_suffix}" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = %{deletion_protection} +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "tf-test-source-profile%{random_suffix}" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} + +data "google_bigquery_default_service_account" "bq_sa" { +} + +resource "google_kms_crypto_key_iam_member" "bigquery_key_user" { + crypto_key_id = "%{bigquery_destination_table_kms_key_name}" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}" +} + +resource "google_datastream_connection_profile" "destination_connection_profile" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "tf-test-destination-profile%{random_suffix}" + + bigquery_profile {} +} + +resource "google_datastream_stream" "default" { + depends_on = [ + google_kms_crypto_key_iam_member.bigquery_key_user + ] + stream_id = "tf-test-my-stream%{random_suffix}" + location = "us-central1" + display_name = "my stream" + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config {} + } + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id + bigquery_destination_config { + source_hierarchy_datasets { + dataset_template { + location = "us-central1" + kms_key_name = "%{bigquery_destination_table_kms_key_name}" + } + } + } + } + + backfill_none { + } } `, context) } diff --git a/website/docs/r/datastream_stream.html.markdown b/website/docs/r/datastream_stream.html.markdown index 9a658217ad6..8fa05edbc2d 100644 --- a/website/docs/r/datastream_stream.html.markdown +++ b/website/docs/r/datastream_stream.html.markdown @@ -133,6 +133,12 @@ resource "google_storage_bucket_iam_member" "reader" { member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" } +resource "google_kms_crypto_key_iam_member" "key_user" { + crypto_key_id = "kms-name" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + resource "google_datastream_connection_profile" "destination_connection_profile" { display_name = "Connection profile" location = "us-central1" @@ -145,6 +151,9 @@ resource "google_datastream_connection_profile" "destination_connection_profile" } resource "google_datastream_stream" "default" { + depends_on = [ + google_kms_crypto_key_iam_member.key_user + ] stream_id = "my-stream" desired_state = "NOT_STARTED" location = "us-central1" @@ -221,6 +230,132 @@ resource "google_datastream_stream" "default" { } } } + + customer_managed_encryption_key = "kms-name" +} +``` +
+ + Open in Cloud Shell + +
+## Example Usage - Datastream Stream Bigquery + + +```hcl +data "google_project" "project" { +} + +resource "google_sql_database_instance" "instance" { + name = "my-instance" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = true +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "source-profile" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} + +data "google_bigquery_default_service_account" "bq_sa" { +} + +resource "google_kms_crypto_key_iam_member" "bigquery_key_user" { + crypto_key_id = "bigquery-kms-name" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}" +} + +resource "google_datastream_connection_profile" "destination_connection_profile" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "destination-profile" + + bigquery_profile {} +} + +resource "google_datastream_stream" "default" { + depends_on = [ + google_kms_crypto_key_iam_member.bigquery_key_user + ] + stream_id = "my-stream" + location = "us-central1" + display_name = "my stream" + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config {} + } + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id + bigquery_destination_config { + source_hierarchy_datasets { + dataset_template { + location = "us-central1" + kms_key_name = "bigquery-kms-name" + } + } + } + } + + backfill_none { + } } ``` @@ -506,6 +641,13 @@ The following arguments are supported: If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _. +* `kms_key_name` - + (Optional) + Describes the Cloud KMS encryption key that will be used to protect destination BigQuery + table. The BigQuery Service Account associated with your project requires access to this + encryption key. i.e. projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}. + See https://cloud.google.com/bigquery/docs/customer-managed-encryption for more information. + - - - @@ -522,6 +664,11 @@ The following arguments are supported: (Optional) Backfill strategy to disable automatic backfill for the Stream's objects. +* `customer_managed_encryption_key` - + (Optional) + A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data + will be encrypted using an internal Stream-specific encryption key provisioned through KMS. + * `project` - (Optional) The ID of the project in which the resource belongs. If it is not provided, the provider project is used.