diff --git a/.changelog/11215.txt b/.changelog/11215.txt new file mode 100644 index 00000000000..02e338979ae --- /dev/null +++ b/.changelog/11215.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +bigquery: added support for value `DELTA_LAKE` to `source_format` in `google_bigquery_table` resource +``` \ No newline at end of file diff --git a/google/services/bigquery/resource_bigquery_table.go b/google/services/bigquery/resource_bigquery_table.go index 1065c16ef5b..30fcbd52387 100644 --- a/google/services/bigquery/resource_bigquery_table.go +++ b/google/services/bigquery/resource_bigquery_table.go @@ -517,7 +517,7 @@ func ResourceBigQueryTable() *schema.Resource { Optional: true, Description: `Please see sourceFormat under ExternalDataConfiguration in Bigquery's public API documentation (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externaldataconfiguration) for supported formats. To use "GOOGLE_SHEETS" the scopes must include "googleapis.com/auth/drive.readonly".`, ValidateFunc: validation.StringInSlice([]string{ - "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "ICEBERG", "DATASTORE_BACKUP", "PARQUET", "ORC", "BIGTABLE", + "CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "ICEBERG", "DATASTORE_BACKUP", "PARQUET", "ORC", "BIGTABLE", "DELTA_LAKE", }, false), }, // SourceURIs [Required] The fully-qualified URIs that point to your data in Google Cloud. diff --git a/google/services/bigquery/resource_bigquery_table_test.go b/google/services/bigquery/resource_bigquery_table_test.go index 9d255928001..4b7c5a8187f 100644 --- a/google/services/bigquery/resource_bigquery_table_test.go +++ b/google/services/bigquery/resource_bigquery_table_test.go @@ -642,6 +642,26 @@ func TestAccBigQueryTable_MaterializedView_NonIncremental_basic(t *testing.T) { }) } +func TestAccBigQueryExternalDataTable_deltaLake(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + + datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10)) + tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccBigQueryTableFromGCSDeltaLake(datasetID, tableID, bucketName), + }, + }, + }) +} + func TestAccBigQueryExternalDataTable_parquet(t *testing.T) { t.Parallel() @@ -2644,6 +2664,44 @@ resource "google_bigquery_table" "test" { `, connectionID, datasetID, bucketName, objectName, tableID, metadataCacheMode, maxStaleness) } +func testAccBigQueryTableFromGCSDeltaLake(datasetID, tableID, bucketName string) string { + return fmt.Sprintf(` +resource "google_bigquery_dataset" "test" { + dataset_id = "%s" +} + +resource "google_storage_bucket" "test" { + name = "%s" + location = "US" + force_destroy = true +} + +# Setup Empty Delta Lake table in Bucket. + +// Upload Metadata File. +resource "google_storage_bucket_object" "metadata" { + name = "_delta_log/00000000000000000000.json" + source = "./test-fixtures/simple/metadata/00000000000000000000.json" + bucket = google_storage_bucket.test.name +} + +resource "google_bigquery_table" "test" { + deletion_protection = false + table_id = "%s" + dataset_id = google_bigquery_dataset.test.dataset_id + external_data_configuration { + autodetect = true + source_format = "DELTA_LAKE" + reference_file_schema_uri = "gs://${google_storage_bucket.test.name}/${google_storage_bucket_object.metadata.name}" + + source_uris = [ + "gs://${google_storage_bucket.test.name}/*", + ] + } +} +`, datasetID, bucketName, tableID) +} + func testAccBigQueryTableFromGCSParquet(datasetID, tableID, bucketName, objectName string) string { return fmt.Sprintf(` resource "google_bigquery_dataset" "test" { diff --git a/google/services/bigquery/test-fixtures/simple/metadata/00000000000000000000.json b/google/services/bigquery/test-fixtures/simple/metadata/00000000000000000000.json new file mode 100644 index 00000000000..000d2e4abe3 --- /dev/null +++ b/google/services/bigquery/test-fixtures/simple/metadata/00000000000000000000.json @@ -0,0 +1,6 @@ +{"commitInfo":{"timestamp":1706278148531,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"country\"]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"5","numOutputBytes":"3045"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"1cbc9537-63eb-4799-8647-2d947ae8fa41"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"1f110132-a652-4be9-815e-348f294515cf","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"first_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"country\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"continent\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["country"],"configuration":{},"createdTime":1706278146762}} +{"add":{"path":"country=Argentina/part-00000-8d0390a3-f797-4265-b9c2-da1c941680a3.c000.snappy.parquet","partitionValues":{"country":"Argentina"},"size":1018,"modificationTime":1706278148083,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Ernesto\",\"last_name\":\"Guevara\",\"continent\":\"NaN\"},\"maxValues\":{\"first_name\":\"Ernesto\",\"last_name\":\"Guevara\",\"continent\":\"NaN\"},\"nullCount\":{\"first_name\":0,\"last_name\":0,\"continent\":0}}"}} +{"add":{"path":"country=China/part-00000-88fba1af-b28d-4303-9c85-9a97be631d40.c000.snappy.parquet","partitionValues":{"country":"China"},"size":1002,"modificationTime":1706278148138,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"first_name\":\"Bruce\",\"last_name\":\"Lee\",\"continent\":\"Asia\"},\"maxValues\":{\"first_name\":\"Jack\",\"last_name\":\"Ma\",\"continent\":\"Asia\"},\"nullCount\":{\"first_name\":0,\"last_name\":0,\"continent\":0}}"}} +{"add":{"path":"country=Germany/part-00000-030076e1-5ec9-47c2-830a-1569f823b6ee.c000.snappy.parquet","partitionValues":{"country":"Germany"},"size":1025,"modificationTime":1706278148185,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"first_name\":\"Soraya\",\"last_name\":\"Jala\",\"continent\":\"NaN\"},\"maxValues\":{\"first_name\":\"Wolfgang\",\"last_name\":\"Manche\",\"continent\":\"NaN\"},\"nullCount\":{\"first_name\":0,\"last_name\":0,\"continent\":0}}"}}