diff --git a/.changelog/7451.txt b/.changelog/7451.txt new file mode 100644 index 00000000000..f1095f464d5 --- /dev/null +++ b/.changelog/7451.txt @@ -0,0 +1,3 @@ +```release-note:bug +datastream: fixed bug where field `dataset_id` could not utilize the id from bigquery directly +``` diff --git a/google/resource_datastream_stream.go b/google/resource_datastream_stream.go index 0672ec6c16e..7558541de44 100644 --- a/google/resource_datastream_stream.go +++ b/google/resource_datastream_stream.go @@ -19,6 +19,7 @@ import ( "fmt" "log" "reflect" + "regexp" "strings" "time" @@ -78,6 +79,20 @@ func waitForDatastreamStreamReady(d *schema.ResourceData, config *Config, timeou }) } +func resourceDatastreamStreamDatabaseIdDiffSuppress(_, old, new string, _ *schema.ResourceData) bool { + re := regexp.MustCompile(`projects/(.+)/datasets/([^\.\?\#]+)`) + paths := re.FindStringSubmatch(new) + + // db returns value in form : + if len(paths) == 3 { + project := paths[1] + datasetId := paths[2] + new = fmt.Sprintf("%s:%s", project, datasetId) + } + + return old == new +} + func ResourceDatastreamStream() *schema.Resource { return &schema.Resource{ Create: resourceDatastreamStreamCreate, @@ -135,9 +150,11 @@ A duration in seconds with up to nine fractional digits, terminated by 's'. Exam Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "dataset_id": { - Type: schema.TypeString, - Required: true, - Description: `Dataset ID in the format projects/{project}/datasets/{dataset_id}`, + Type: schema.TypeString, + Required: true, + DiffSuppressFunc: resourceDatastreamStreamDatabaseIdDiffSuppress, + Description: `Dataset ID in the format projects/{project}/datasets/{dataset_id} or +{project}:{dataset_id}`, }, }, }, @@ -5086,7 +5103,16 @@ func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSingleTarge } func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSingleTargetDatasetDatasetId(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { - return v, nil + s := v.(string) + re := regexp.MustCompile(`projects/(.+)/datasets/([^\.\?\#]+)`) + paths := re.FindStringSubmatch(s) + if len(paths) == 3 { + project := paths[1] + datasetId := paths[2] + return fmt.Sprintf("%s:%s", project, datasetId), nil + } + + return s, nil } func expandDatastreamStreamDestinationConfigBigqueryDestinationConfigSourceHierarchyDatasets(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { diff --git a/google/resource_datastream_stream_generated_test.go b/google/resource_datastream_stream_generated_test.go index 638777769c2..91a4ff73375 100644 --- a/google/resource_datastream_stream_generated_test.go +++ b/google/resource_datastream_stream_generated_test.go @@ -415,6 +415,147 @@ resource "google_datastream_stream" "default" { `, context) } +func TestAccDatastreamStream_datastreamStreamPostgresqlBigqueryDatasetIdExample(t *testing.T) { + SkipIfVcr(t) + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": RandString(t, 10), + } + + VcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: TestAccProviders, + ExternalProviders: map[string]resource.ExternalProvider{ + "random": {}, + "time": {}, + }, + CheckDestroy: testAccCheckDatastreamStreamDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDatastreamStream_datastreamStreamPostgresqlBigqueryDatasetIdExample(context), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + }, + }) +} + +func testAccDatastreamStream_datastreamStreamPostgresqlBigqueryDatasetIdExample(context map[string]interface{}) string { + return Nprintf(` + +resource "google_bigquery_dataset" "postgres" { + dataset_id = "postgres%{random_suffix}" + friendly_name = "postgres" + description = "Database of postgres" + location = "us-central1" +} + +resource "google_datastream_stream" "default" { + display_name = "postgres to bigQuery" + location = "us-central1" + stream_id = "postgres-to-big-query%{random_suffix}" + + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config {} + } + + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile2.id + bigquery_destination_config { + data_freshness = "900s" + single_target_dataset { + dataset_id = google_bigquery_dataset.postgres.id + } + } + } + + backfill_all { + } + +} + +resource "google_datastream_connection_profile" "destination_connection_profile2" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "tf-test-destination-profile%{random_suffix}" + bigquery_profile {} +} + +resource "google_sql_database_instance" "instance" { + name = "tf-test-my-instance%{random_suffix}" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = false +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user%{random_suffix}" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "tf-test-source-profile%{random_suffix}" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} +`, context) +} + func TestAccDatastreamStream_datastreamStreamBigqueryExample(t *testing.T) { SkipIfVcr(t) t.Parallel() diff --git a/website/docs/r/datastream_stream.html.markdown b/website/docs/r/datastream_stream.html.markdown index 533fde9d609..cc1e36bc7e5 100644 --- a/website/docs/r/datastream_stream.html.markdown +++ b/website/docs/r/datastream_stream.html.markdown @@ -413,6 +413,122 @@ resource "google_datastream_stream" "stream5" { } } ``` +
+ + Open in Cloud Shell + +
+## Example Usage - Datastream Stream Postgresql Bigquery Dataset + + +```hcl + +resource "google_bigquery_dataset" "postgres" { + dataset_id = "postgres%{random_suffix}" + friendly_name = "postgres" + description = "Database of postgres" + location = "us-central1" +} + +resource "google_datastream_stream" "default" { + display_name = "postgres to bigQuery" + location = "us-central1" + stream_id = "postgres-to-big-query%{random_suffix}" + + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config {} + } + + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile2.id + bigquery_destination_config { + data_freshness = "900s" + single_target_dataset { + dataset_id = google_bigquery_dataset.postgres.id + } + } + } + + backfill_all { + } + +} + +resource "google_datastream_connection_profile" "destination_connection_profile2" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "tf-test-destination-profile%{random_suffix}" + bigquery_profile {} +} + +resource "google_sql_database_instance" "instance" { + name = "tf-test-my-instance%{random_suffix}" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = false +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user%{random_suffix}" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "tf-test-source-profile%{random_suffix}" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} +```
Open in Cloud Shell @@ -1145,7 +1261,8 @@ The following arguments are supported: * `dataset_id` - (Required) - Dataset ID in the format projects/{project}/datasets/{dataset_id} + Dataset ID in the format projects/{project}/datasets/{dataset_id} or + {project}:{dataset_id} The `source_hierarchy_datasets` block supports: