diff --git a/mmv1/products/pubsub/api.yaml b/mmv1/products/pubsub/api.yaml index 8ad74bc6367c..b3d3b6a2a3b1 100644 --- a/mmv1/products/pubsub/api.yaml +++ b/mmv1/products/pubsub/api.yaml @@ -148,8 +148,38 @@ objects: name: 'labels' description: | A set of key/value label pairs to assign to this Subscription. + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryConfig' + conflicts: + - push_config + description: | + If delivery to BigQuery is used with this subscription, this field is used to configure it. + Either pushConfig or bigQueryConfig can be set, but not both. + If both are empty, then the subscriber will pull and ack messages using API methods. + properties: + - !ruby/object:Api::Type::String + name: 'table' + description: | + The name of the table to which to write data, of the form {projectId}.{datasetId}.{tableId} + required: true + - !ruby/object:Api::Type::Boolean + name: 'useTopicSchema' + description: | + When true, use the topic's schema as the columns to write to in BigQuery, if it exists. + - !ruby/object:Api::Type::Boolean + name: 'writeMetadata' + description: | + When true, write the subscription name, messageId, publishTime, attributes, and orderingKey to additional columns in the table. + The subscription name, messageId, and publishTime fields are put in their own columns while all other message properties (other than data) are written to a JSON object in the attributes column. + - !ruby/object:Api::Type::Boolean + name: 'dropUnknownFields' + description: | + When true and useTopicSchema is true, any fields that are a part of the topic schema that are not part of the BigQuery table schema are dropped when writing to BigQuery. + Otherwise, the schemas must be kept in sync and any messages with extra fields are not written and remain in the subscription's backlog. - !ruby/object:Api::Type::NestedObject name: 'pushConfig' + conflicts: + - bigquery_config description: | If push delivery is used with this subscription, this field is used to configure it. An empty pushConfig signifies that the subscriber will diff --git a/mmv1/products/pubsub/terraform.yaml b/mmv1/products/pubsub/terraform.yaml index 02a59c6ea384..59b6839df17f 100644 --- a/mmv1/products/pubsub/terraform.yaml +++ b/mmv1/products/pubsub/terraform.yaml @@ -111,6 +111,14 @@ overrides: !ruby/object:Overrides::ResourceOverrides vars: topic_name: "example-topic" subscription_name: "example-subscription" + - !ruby/object:Provider::Terraform::Examples + name: "pubsub_subscription_push_bq" + primary_resource_id: "example" + vars: + topic_name: "example-topic" + subscription_name: "example-subscription" + dataset_id: "example_dataset" + table_id: "example_table" docs: !ruby/object:Provider::Terraform::Docs note: | You can retrieve the email of the Google Managed Pub/Sub Service Account used for forwarding diff --git a/mmv1/templates/terraform/examples/pubsub_subscription_push_bq.tf.erb b/mmv1/templates/terraform/examples/pubsub_subscription_push_bq.tf.erb new file mode 100644 index 000000000000..9bc5915278d4 --- /dev/null +++ b/mmv1/templates/terraform/examples/pubsub_subscription_push_bq.tf.erb @@ -0,0 +1,50 @@ +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['topic_name'] %>" +} + +resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['subscription_name'] %>" + topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.name + + bigquery_config { + table = "${google_bigquery_table.test.project}.${google_bigquery_table.test.dataset_id}.${google_bigquery_table.test.table_id}" + } + + depends_on = [google_project_iam_member.viewer, google_project_iam_member.editor] +} + +data "google_project" "project" { +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_bigquery_dataset" "test" { + dataset_id = "<%= ctx[:vars]['dataset_id'] %>" +} + +resource "google_bigquery_table" "test" { + deletion_protection = false + table_id = "<%= ctx[:vars]['table_id'] %>" + dataset_id = google_bigquery_dataset.test.dataset_id + + schema = <