From 2efa7e735985666689f4d70becc9b25769955e7a Mon Sep 17 00:00:00 2001 From: Kamal Aboul-Hosn Date: Fri, 14 Jun 2024 10:53:39 -0400 Subject: [PATCH] feat: Add Pub/Sub Subscription support for specifying a service account (#10967) --- mmv1/products/pubsub/Subscription.yaml | 29 +++ ...ubscription_push_bq_service_account.tf.erb | 56 ++++++ ...n_push_cloudstorage_service_account.tf.erb | 46 +++++ .../resource_pubsub_subscription_test.go | 168 ++++++++++++++++-- 4 files changed, 286 insertions(+), 13 deletions(-) create mode 100644 mmv1/templates/terraform/examples/pubsub_subscription_push_bq_service_account.tf.erb create mode 100644 mmv1/templates/terraform/examples/pubsub_subscription_push_cloudstorage_service_account.tf.erb diff --git a/mmv1/products/pubsub/Subscription.yaml b/mmv1/products/pubsub/Subscription.yaml index 97a17de701cd..d70884b88cfa 100644 --- a/mmv1/products/pubsub/Subscription.yaml +++ b/mmv1/products/pubsub/Subscription.yaml @@ -64,6 +64,15 @@ examples: subscription_name: 'example-subscription' dataset_id: 'example_dataset' table_id: 'example_table' + - !ruby/object:Provider::Terraform::Examples + name: 'pubsub_subscription_push_bq_service_account' + primary_resource_id: 'example' + vars: + topic_name: 'example-topic' + subscription_name: 'example-subscription' + dataset_id: 'example_dataset' + table_id: 'example_table' + service_account_id: 'example-bqw' - !ruby/object:Provider::Terraform::Examples name: 'pubsub_subscription_push_cloudstorage' primary_resource_id: 'example' @@ -78,6 +87,14 @@ examples: topic_name: 'example-topic' subscription_name: 'example-subscription' bucket_name: 'example-bucket' + - !ruby/object:Provider::Terraform::Examples + name: 'pubsub_subscription_push_cloudstorage_service_account' + primary_resource_id: 'example' + vars: + topic_name: 'example-topic' + subscription_name: 'example-subscription' + bucket_name: 'example-bucket' + service_account_id: 'example-stw' docs: !ruby/object:Provider::Terraform::Docs note: | You can retrieve the email of the Google Managed Pub/Sub Service Account used for forwarding @@ -150,6 +167,12 @@ properties: When true and use_topic_schema or use_table_schema is true, any fields that are a part of the topic schema or message schema that are not part of the BigQuery table schema are dropped when writing to BigQuery. Otherwise, the schemas must be kept in sync and any messages with extra fields are not written and remain in the subscription's backlog. + - !ruby/object:Api::Type::String + name: 'serviceAccountEmail' + description: | + The service account to use to write to BigQuery. If not specified, the Pub/Sub + [service agent](https://cloud.google.com/iam/docs/service-agents), + service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com, is used. - !ruby/object:Api::Type::NestedObject name: 'cloudStorageConfig' conflicts: @@ -207,6 +230,12 @@ properties: name: 'writeMetadata' description: | When true, write the subscription name, messageId, publishTime, attributes, and orderingKey as additional fields in the output. + - !ruby/object:Api::Type::String + name: 'serviceAccountEmail' + description: | + The service account to use to write to Cloud Storage. If not specified, the Pub/Sub + [service agent](https://cloud.google.com/iam/docs/service-agents), + service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com, is used. - !ruby/object:Api::Type::NestedObject name: 'pushConfig' conflicts: diff --git a/mmv1/templates/terraform/examples/pubsub_subscription_push_bq_service_account.tf.erb b/mmv1/templates/terraform/examples/pubsub_subscription_push_bq_service_account.tf.erb new file mode 100644 index 000000000000..9dc8061980cb --- /dev/null +++ b/mmv1/templates/terraform/examples/pubsub_subscription_push_bq_service_account.tf.erb @@ -0,0 +1,56 @@ +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['topic_name'] %>" +} + +resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['subscription_name'] %>" + topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.id + + bigquery_config { + table = "${google_bigquery_table.test.project}.${google_bigquery_table.test.dataset_id}.${google_bigquery_table.test.table_id}" + service_account_email = google_service_account.bq_write_service_account.email + } + + depends_on = [google_service_account.bq_write_service_account, google_project_iam_member.viewer, google_project_iam_member.editor] +} + +data "google_project" "project" { +} + +resource "google_service_account" "bq_write_service_account" { + account_id = "<%= ctx[:vars]['service_account_id'] %>" + display_name = "BQ Write Service Account" +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +} + +resource "google_bigquery_dataset" "test" { + dataset_id = "<%= ctx[:vars]['dataset_id'] %>" +} + +resource "google_bigquery_table" "test" { + deletion_protection = false + table_id = "<%= ctx[:vars]['table_id'] %>" + dataset_id = google_bigquery_dataset.test.dataset_id + + schema = <" { + name = "<%= ctx[:vars]['bucket_name'] %>" + location = "US" + uniform_bucket_level_access = true +} + +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['topic_name'] %>" +} + +resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['subscription_name'] %>" + topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.id + + cloud_storage_config { + bucket = google_storage_bucket.<%= ctx[:primary_resource_id] %>.name + + filename_prefix = "pre-" + filename_suffix = "-%{random_suffix}" + filename_datetime_format = "YYYY-MM-DD/hh_mm_ssZ" + + max_bytes = 1000 + max_duration = "300s" + + service_account_email = google_service_account.storage_write_service_account.email + } + depends_on = [ + google_service_account.storage_write_service_account, + google_storage_bucket.<%= ctx[:primary_resource_id] %>, + google_storage_bucket_iam_member.admin, + ] +} + +data "google_project" "project" { +} + +resource "google_service_account" "storage_write_service_account" { + account_id = "<%= ctx[:vars]['service_account_id'] %>" + display_name = "Storage Write Service Account" +} + +resource "google_storage_bucket_iam_member" "admin" { + bucket = google_storage_bucket.<%= ctx[:primary_resource_id] %>.name + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.storage_write_service_account.email}" +} diff --git a/mmv1/third_party/terraform/services/pubsub/resource_pubsub_subscription_test.go b/mmv1/third_party/terraform/services/pubsub/resource_pubsub_subscription_test.go index 4ae710112c1c..e368b8d1fa35 100644 --- a/mmv1/third_party/terraform/services/pubsub/resource_pubsub_subscription_test.go +++ b/mmv1/third_party/terraform/services/pubsub/resource_pubsub_subscription_test.go @@ -185,7 +185,7 @@ func TestAccPubsubSubscriptionBigQuery_update(t *testing.T) { CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, false), + Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, false, ""), }, { ResourceName: "google_pubsub_subscription.foo", @@ -194,7 +194,51 @@ func TestAccPubsubSubscriptionBigQuery_update(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, true), + Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, true, ""), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccPubsubSubscriptionBigQuery_serviceAccount(t *testing.T) { + t.Parallel() + + dataset := fmt.Sprintf("tftestdataset%s", acctest.RandString(t, 10)) + table := fmt.Sprintf("tf-test-table-%s", acctest.RandString(t, 10)) + topic := fmt.Sprintf("tf-test-topic-%s", acctest.RandString(t, 10)) + subscriptionShort := fmt.Sprintf("tf-test-sub-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, false, "bq-test-sa"), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, true, ""), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscriptionShort, true, "bq-test-sa2"), }, { ResourceName: "google_pubsub_subscription.foo", @@ -219,7 +263,50 @@ func TestAccPubsubSubscriptionCloudStorage_update(t *testing.T) { CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "", "", "", 0, ""), + Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "", "", "", 0, "", ""), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "pre-", "-suffix", "YYYY-MM-DD/hh_mm_ssZ", 1000, "300s", ""), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccPubsubSubscriptionCloudStorage_serviceAccount(t *testing.T) { + t.Parallel() + + bucket := fmt.Sprintf("tf-test-bucket-%s", acctest.RandString(t, 10)) + topic := fmt.Sprintf("tf-test-topic-%s", acctest.RandString(t, 10)) + subscriptionShort := fmt.Sprintf("tf-test-sub-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "", "", "", 0, "", "gcs-test-sa"), + }, + { + ResourceName: "google_pubsub_subscription.foo", + ImportStateId: subscriptionShort, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "pre-", "-suffix", "YYYY-MM-DD/hh_mm_ssZ", 1000, "300s", ""), }, { ResourceName: "google_pubsub_subscription.foo", @@ -228,7 +315,7 @@ func TestAccPubsubSubscriptionCloudStorage_update(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "pre-", "-suffix", "YYYY-MM-DD/hh_mm_ssZ", 1000, "300s"), + Config: testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscriptionShort, "", "", "", 0, "", "gcs-test-sa2"), }, { ResourceName: "google_pubsub_subscription.foo", @@ -435,10 +522,30 @@ resource "google_pubsub_subscription" "foo" { `, topic, subscription, label, deadline, exactlyOnceDelivery) } -func testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscription string, useTableSchema bool) string { - return fmt.Sprintf(` -data "google_project" "project" { } +func testAccPubsubSubscriptionBigQuery_basic(dataset, table, topic, subscription string, useTableSchema bool, serviceAccountId string) string { + serivceAccountEmailField := "" + serivceAccountResource := "" + if serviceAccountId != "" { + serivceAccountResource = fmt.Sprintf(` +resource "google_service_account" "bq_write_service_account" { + account_id = "%s" + display_name = "BQ Write Service Account" +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +} +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.bq_write_service_account.email}" +}`, serviceAccountId) + serivceAccountEmailField = "service_account_email = google_service_account.bq_write_service_account.email" + } else { + serivceAccountResource = fmt.Sprintf(` resource "google_project_iam_member" "viewer" { project = data.google_project.project.project_id role = "roles/bigquery.metadataViewer" @@ -450,6 +557,13 @@ resource "google_project_iam_member" "editor" { role = "roles/bigquery.dataEditor" member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" } + `) + } + + return fmt.Sprintf(` +data "google_project" "project" { } + +%s resource "google_bigquery_dataset" "test" { dataset_id = "%s" @@ -483,6 +597,7 @@ resource "google_pubsub_subscription" "foo" { bigquery_config { table = "${google_bigquery_table.test.project}.${google_bigquery_table.test.dataset_id}.${google_bigquery_table.test.table_id}" use_table_schema = %t + %s } depends_on = [ @@ -490,10 +605,10 @@ resource "google_pubsub_subscription" "foo" { google_project_iam_member.editor ] } -`, dataset, table, topic, subscription, useTableSchema) + `, serivceAccountResource, dataset, table, topic, subscription, useTableSchema, serivceAccountEmailField) } -func testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscription, filenamePrefix, filenameSuffix, filenameDatetimeFormat string, maxBytes int, maxDuration string) string { +func testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscription, filenamePrefix, filenameSuffix, filenameDatetimeFormat string, maxBytes int, maxDuration string, serviceAccountId string) string { filenamePrefixString := "" if filenamePrefix != "" { filenamePrefixString = fmt.Sprintf(`filename_prefix = "%s"`, filenamePrefix) @@ -514,20 +629,46 @@ func testAccPubsubSubscriptionCloudStorage_basic(bucket, topic, subscription, fi if maxDuration != "" { maxDurationString = fmt.Sprintf(`max_duration = "%s"`, maxDuration) } - return fmt.Sprintf(` -data "google_project" "project" { } + + serivceAccountEmailField := "" + serivceAccountResource := "" + if serviceAccountId != "" { + serivceAccountResource = fmt.Sprintf(` +resource "google_service_account" "storage_write_service_account" { + account_id = "%s" + display_name = "Write Service Account" +} resource "google_storage_bucket_iam_member" "admin" { bucket = google_storage_bucket.test.name role = "roles/storage.admin" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" + member = "serviceAccount:${google_service_account.storage_write_service_account.email}" } +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.storage_write_service_account.email}" +}`, serviceAccountId) + serivceAccountEmailField = "service_account_email = google_service_account.storage_write_service_account.email" + } else { + serivceAccountResource = fmt.Sprintf(` +resource "google_storage_bucket_iam_member" "admin" { + bucket = google_storage_bucket.test.name + role = "roles/storage.admin" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +}`) + } + return fmt.Sprintf(` +data "google_project" "project" { } + resource "google_storage_bucket" "test" { name = "%s" location = "US" } +%s + resource "google_pubsub_topic" "foo" { name = "%s" } @@ -543,6 +684,7 @@ resource "google_pubsub_subscription" "foo" { %s %s %s + %s } depends_on = [ @@ -550,7 +692,7 @@ resource "google_pubsub_subscription" "foo" { google_storage_bucket_iam_member.admin, ] } -`, bucket, topic, subscription, filenamePrefixString, filenameSuffixString, filenameDatetimeString, maxBytesString, maxDurationString) +`, bucket, serivceAccountResource, topic, subscription, filenamePrefixString, filenameSuffixString, filenameDatetimeString, maxBytesString, maxDurationString, serivceAccountEmailField) } func testAccPubsubSubscription_topicOnly(topic string) string {