hashicorp · modular-magician · Jul 7, 2023 · Jul 7, 2023
diff --git a/.changelog/7128.txt b/.changelog/7128.txt
@@ -0,0 +1,6 @@
+```release-note:enhancement
+bigquery: table - add `external_data_configuration.metadata_cache_mode` and `external_data_configuration.object_metadata`
+```
+```release-note:enhancement
+bigquery: table - make `external_data_configuration.source_fomat` optional
+```
diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go
@@ -482,6 +482,34 @@ func TestAccBigQueryExternalDataTable_parquet(t *testing.T) {
 	})
 }
 
+func TestAccBigQueryExternalDataTable_objectTable(t *testing.T) {
+	t.Parallel()
+
+	bucketName := testBucketName(t)
+	objectName := fmt.Sprintf("tf_test_%s.csv", RandString(t, 10))
+
+	datasetID := fmt.Sprintf("tf_test_%s", RandString(t, 10))
+	tableID := fmt.Sprintf("tf_test_%s", RandString(t, 10))
+	connectionID := fmt.Sprintf("tf_test_%s", RandString(t, 10))
+
+	VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckBigQueryTableDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccBigQueryTableFromGCSObjectTable(connectionID, datasetID, tableID, bucketName, objectName),
+			},
+			{
+				Config: testAccBigQueryTableFromGCSObjectTableMetadata(connectionID, datasetID, tableID, bucketName, objectName),
+			},
+			{
+				Config: testAccBigQueryTableFromGCSObjectTable(connectionID, datasetID, tableID, bucketName, objectName),
+			},
+		},
+	})
+}
+
 func TestAccBigQueryExternalDataTable_CSV(t *testing.T) {
 	t.Parallel()
 
@@ -1555,6 +1583,115 @@ resource "google_bigquery_table" "test" {
 `, datasetID, bucketName, objectName, tableID)
 }
 
+func testAccBigQueryTableFromGCSObjectTable(connectionID, datasetID, tableID, bucketName, objectName string) string {
+	return fmt.Sprintf(`
+resource "google_bigquery_connection" "test" {
+   connection_id = "%s"
+   location = "US"
+   cloud_resource {}
+}
+
+locals {
+   connection_id_split = split("/", google_bigquery_connection.test.name)
+   connection_id_reformatted = "${local.connection_id_split[1]}.${local.connection_id_split[3]}.${local.connection_id_split[5]}"
+}
+
+data "google_project" "project" {}
+
+resource "google_project_iam_member" "test" {
+   role = "roles/storage.objectViewer"
+   project = data.google_project.project.id
+   member = "serviceAccount:${google_bigquery_connection.test.cloud_resource[0].service_account_id}"
+}
+
+resource "google_bigquery_dataset" "test" {
+  dataset_id = "%s"
+}
+
+resource "google_storage_bucket" "test" {
+  name          = "%s"
+  location      = "US"
+  force_destroy = true
+}
+
+resource "google_storage_bucket_object" "test" {
+  name    = "%s"
+  source = "./test-fixtures/bigquerytable/test.parquet.gzip"
+  bucket = google_storage_bucket.test.name
+}
+
+resource "google_bigquery_table" "test" {
+  deletion_protection = false
+  table_id   = "%s"
+  dataset_id = google_bigquery_dataset.test.dataset_id
+  external_data_configuration {
+	connection_id   = local.connection_id_reformatted
+    autodetect      = false
+	object_metadata = "SIMPLE"
+
+    source_uris = [
+      "gs://${google_storage_bucket.test.name}/*",
+    ]
+  }
+}
+`, connectionID, datasetID, bucketName, objectName, tableID)
+}
+
+func testAccBigQueryTableFromGCSObjectTableMetadata(connectionID, datasetID, tableID, bucketName, objectName string) string {
+	return fmt.Sprintf(`
+resource "google_bigquery_connection" "test" {
+   connection_id = "%s"
+   location = "US"
+   cloud_resource {}
+}
+
+locals {
+   connection_id_split = split("/", google_bigquery_connection.test.name)
+   connection_id_reformatted = "${local.connection_id_split[1]}.${local.connection_id_split[3]}.${local.connection_id_split[5]}"
+}
+
+data "google_project" "project" {}
+
+resource "google_project_iam_member" "test" {
+   role = "roles/storage.objectViewer"
+   project = data.google_project.project.id
+   member = "serviceAccount:${google_bigquery_connection.test.cloud_resource[0].service_account_id}"
+}
+
+resource "google_bigquery_dataset" "test" {
+  dataset_id = "%s"
+}
+
+resource "google_storage_bucket" "test" {
+  name          = "%s"
+  location      = "US"
+  force_destroy = true
+}
+
+resource "google_storage_bucket_object" "test" {
+  name    = "%s"
+  source = "./test-fixtures/bigquerytable/test.parquet.gzip"
+  bucket = google_storage_bucket.test.name
+}
+
+resource "google_bigquery_table" "test" {
+  deletion_protection = false
+  table_id   = "%s"
+  dataset_id = google_bigquery_dataset.test.dataset_id
+  external_data_configuration {
+	connection_id       = local.connection_id_reformatted
+    autodetect          = false
+	object_metadata     = "SIMPLE"
+	metadata_cache_mode = "MANUAL"
+
+    source_uris = [
+      "gs://${google_storage_bucket.test.name}/*",
+    ]
+  }
+}
+`, connectionID, datasetID, bucketName, objectName, tableID)
+}
+
 func testAccBigQueryTableFromGCSWithSchemaWithConnectionId(datasetID, tableID, connectionID, projectID, bucketName, objectName, content, schema string) string {
 	return fmt.Sprintf(`
 resource "google_bigquery_dataset" "test" {

diff --git a/google/services/bigquery/resource_bigquery_table.go b/google/services/bigquery/resource_bigquery_table.go
@@ -444,7 +444,7 @@ func ResourceBigQueryTable() *schema.Resource {
 						// SourceFormat [Required] The data format.
 						"source_format": {
 							Type:        schema.TypeString,
-							Required:    true,
+							Optional:    true,
 							Description: ` Please see sourceFormat under ExternalDataConfiguration in Bigquery's public API documentation (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externaldataconfiguration) for supported formats. To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`,
 							ValidateFunc: validation.StringInSlice([]string{
 								"CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "ICEBERG", "DATASTORE_BACKUP", "PARQUET", "ORC", "BIGTABLE",
@@ -660,6 +660,18 @@ func ResourceBigQueryTable() *schema.Resource {
 							Optional:    true,
 							Description: `When creating an external table, the user can provide a reference file with the table schema. This is enabled for the following formats: AVRO, PARQUET, ORC.`,
 						},
+						"metadata_cache_mode": {
+							Type:         schema.TypeString,
+							Optional:     true,
+							Description:  `Metadata Cache Mode for the table. Set this to enable caching of metadata from external data source.`,
+							ValidateFunc: validation.StringInSlice([]string{"AUTOMATIC", "MANUAL"}, false),
+						},
+						"object_metadata": {
+							Type:          schema.TypeString,
+							Optional:      true,
+							Description:   `Object Metadata is used to create Object Tables. Object Tables contain a listing of objects (with their metadata) found at the sourceUris. If ObjectMetadata is set, sourceFormat should be omitted.`,
+							ConflictsWith: []string{"external_data_configuration.0.source_format"},
+						},
 					},
 				},
 			},
@@ -1394,6 +1406,12 @@ func expandExternalDataConfiguration(cfg interface{}) (*bigquery.ExternalDataCon
 	if v, ok := raw["reference_file_schema_uri"]; ok {
 		edc.ReferenceFileSchemaUri = v.(string)
 	}
+	if v, ok := raw["metadata_cache_mode"]; ok {
+		edc.MetadataCacheMode = v.(string)
+	}
+	if v, ok := raw["object_metadata"]; ok {
+		edc.ObjectMetadata = v.(string)
+	}
 
 	return edc, nil
 
@@ -1443,6 +1461,13 @@ func flattenExternalDataConfiguration(edc *bigquery.ExternalDataConfiguration) (
 	if edc.ReferenceFileSchemaUri != "" {
 		result["reference_file_schema_uri"] = edc.ReferenceFileSchemaUri
 	}
+	if edc.MetadataCacheMode != "" {
+		result["metadata_cache_mode"] = edc.MetadataCacheMode
+	}
+
+	if edc.ObjectMetadata != "" {
+		result["object_metadata"] = edc.ObjectMetadata
+	}
 
 	return []map[string]interface{}{result}, nil
 }

diff --git a/website/docs/r/bigquery_table.html.markdown b/website/docs/r/bigquery_table.html.markdown
@@ -195,7 +195,7 @@ in Terraform state, a `terraform destroy` or `terraform apply` that would delete
     datasource, after creation the computed schema will be stored in
     `google_bigquery_table.schema`
 
-* `source_format` (Required) - The data format. Please see sourceFormat under
+* `source_format` (Optional) - The data format. Please see sourceFormat under
     [ExternalDataConfiguration](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externaldataconfiguration)
     in Bigquery's public API documentation for supported formats. To use "GOOGLE_SHEETS"
     the `scopes` must include "https://www.googleapis.com/auth/drive.readonly".
@@ -205,6 +205,10 @@ in Terraform state, a `terraform destroy` or `terraform apply` that would delete
 
 * `reference_file_schema_uri` - (Optional) When creating an external table, the user can provide a reference file with the table schema. This is enabled for the following formats: AVRO, PARQUET, ORC.
 
+* `metadata_cache_mode` - (Optional) Metadata Cache Mode for the table. Set this to enable caching of metadata from external data source. Valid values are `AUTOMATIC` and `MANUAL`.
+
+* `object_metadata` - (Optional) Object Metadata is used to create Object Tables. Object Tables contain a listing of objects (with their metadata) found at the sourceUris. If `object_metadata` is set, `source_format` should be omitted.
+
 <a name="nested_csv_options"></a>The `csv_options` block supports:
 
 * `quote` (Required) - The value that is used to quote data sections in a