Skip to content

Commit

Permalink
Handle unsetting policy tags in BigQuery table schema (GoogleCloudPla…
Browse files Browse the repository at this point in the history 
…tform#8658)
  • Loading branch information
@wj-chen @joelkattapuram
wj-chen authored and joelkattapuram committed Sep 20, 2023
1 parent eaea283 commit c964aa6
Show file tree
Hide file tree
Showing 2 changed files with 319 additions and 3 deletions.
51 changes: 48 additions & 3 deletions mmv1/third_party/terraform/services/bigquery/resource_bigquery_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ func bigQueryTableMapKeyOverride(key string, objectA, objectB map[string]interfa
return false
}
return bigQueryTableTypeEq(valA.(string), valB.(string))
case "policyTags":
eq := bigQueryTableNormalizePolicyTags(valA) == nil && bigQueryTableNormalizePolicyTags(valB) == nil
return eq
}

// otherwise rely on default behavior
Expand Down Expand Up @@ -223,6 +226,23 @@ func bigQueryTableModeIsForceNew(old, new string) bool {
return !eq && !reqToNull
}

func bigQueryTableNormalizePolicyTags(val interface{}) interface{} {
if val == nil {
return nil
}
if policyTags, ok := val.(map[string]interface{}); ok {
// policyTags = {} is same as nil.
if len(policyTags) == 0 {
return nil
}
// policyTags = {names = []} is same as nil.
if names, ok := policyTags["names"].([]interface{}); ok && len(names) == 0 {
return nil
}
}
return val
}

// Compares two existing schema implementations and decides if
// it is changeable.. pairs with a force new on not changeable
func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error) {
Expand Down Expand Up @@ -1112,7 +1132,10 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e
}

if v, ok := d.GetOk("schema"); ok {
schema, err := expandSchema(v)
_, viewPresent := d.GetOk("view")
_, materializedViewPresent := d.GetOk("materialized_view")
managePolicyTags := !viewPresent && !materializedViewPresent
schema, err := expandSchema(v, managePolicyTags)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1460,7 +1483,8 @@ func expandExternalDataConfiguration(cfg interface{}) (*bigquery.ExternalDataCon
edc.MaxBadRecords = int64(v.(int))
}
if v, ok := raw["schema"]; ok {
schema, err := expandSchema(v)
managePolicyTags := true
schema, err := expandSchema(v, managePolicyTags)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1779,7 +1803,7 @@ func flattenJsonOptions(opts *bigquery.JsonOptions) []map[string]interface{} {
return []map[string]interface{}{result}
}

func expandSchema(raw interface{}) (*bigquery.TableSchema, error) {
func expandSchema(raw interface{}, managePolicyTags bool) (*bigquery.TableSchema, error) {
var fields []*bigquery.TableFieldSchema

if len(raw.(string)) == 0 {
Expand All @@ -1790,6 +1814,12 @@ func expandSchema(raw interface{}) (*bigquery.TableSchema, error) {
return nil, err
}

if managePolicyTags {
for _, field := range fields {
setEmptyPolicyTagsInSchema(field)
}
}

return &bigquery.TableSchema{Fields: fields}, nil
}

Expand All @@ -1802,6 +1832,21 @@ func flattenSchema(tableSchema *bigquery.TableSchema) (string, error) {
return string(schema), nil
}

// Explicitly set empty PolicyTags unless the PolicyTags field is specified in the schema.
func setEmptyPolicyTagsInSchema(field *bigquery.TableFieldSchema) {
// Field has children fields.
if len(field.Fields) > 0 {
for _, subField := range field.Fields {
setEmptyPolicyTagsInSchema(subField)
}
return
}
// Field is a leaf.
if field.PolicyTags == nil {
field.PolicyTags = &bigquery.TableFieldSchemaPolicyTags{Names: []string{}}
}
}

func expandTimePartitioning(configured interface{}) *bigquery.TimePartitioning {
raw := configured.([]interface{})[0].(map[string]interface{})
tp := &bigquery.TimePartitioning{Type: raw["type"].(string)}
Expand Down
271 changes: 271 additions & 0 deletions mmv1/third_party/terraform/services/bigquery/resource_bigquery_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -989,6 +989,142 @@ func TestAccBigQueryTable_emptySchema(t *testing.T) {
})
}

func TestAccBigQueryTable_Update_SchemaWithoutPolicyTagsToWithPolicyTags(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
projectID := envvar.GetTestProjectFromEnv()

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableBasicSchema(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccBigQueryTableBasicSchemaWithPolicyTags(datasetID, tableID, projectID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccBigQueryTable_Update_SchemaWithPolicyTagsToNoPolicyTag(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
projectID := envvar.GetTestProjectFromEnv()

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableBasicSchemaWithPolicyTags(datasetID, tableID, projectID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccBigQueryTableBasicSchema(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccBigQueryTable_Update_SchemaWithPolicyTagsToEmptyPolicyTag(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
projectID := envvar.GetTestProjectFromEnv()

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableBasicSchemaWithPolicyTags(datasetID, tableID, projectID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccBigQueryTableBasicSchemaWithEmptyPolicyTags(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccBigQueryTable_Update_SchemaWithPolicyTagsToEmptyPolicyTagNames(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(t, 10))
projectID := envvar.GetTestProjectFromEnv()

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableBasicSchemaWithPolicyTags(datasetID, tableID, projectID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccBigQueryTableBasicSchemaWithEmptyPolicyTagNames(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func testAccCheckBigQueryExtData(t *testing.T, expectedQuoteChar string) resource.TestCheckFunc {
return func(s *terraform.State) error {
for _, rs := range s.RootModule().Resources {
Expand Down Expand Up @@ -1035,6 +1171,141 @@ func testAccCheckBigQueryTableDestroyProducer(t *testing.T) func(s *terraform.St
}
}

func testAccBigQueryTableBasicSchema(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
schema = <<EOH
[
{
"name": "id",
"type": "INTEGER"
}
]
EOH
}
`, datasetID, tableID)
}

func testAccBigQueryTableBasicSchemaWithPolicyTags(datasetID, tableID, projectID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
schema = <<EOH
[
{
"name": "id",
"type": "INTEGER",
"policyTags": {
"names": [
"projects/%s/locations/us/taxonomies/123/policyTags/1"
]
}
},
{
"name": "city",
"type": "RECORD",
"fields": [
{
"name": "id",
"type": "INTEGER",
"policyTags": {
"names": [
"projects/%s/locations/us/taxonomies/123/policyTags/1"
]
}
},
{
"name": "coord",
"type": "RECORD",
"fields": [
{
"name": "lon",
"type": "FLOAT",
"policyTags": {
"names": [
"projects/%s/locations/us/taxonomies/123/policyTags/1"
]
}
}
]
}
]
}
]
EOH
}
`, datasetID, tableID, projectID, projectID, projectID)
}

func testAccBigQueryTableBasicSchemaWithEmptyPolicyTags(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
schema = <<EOH
[
{
"name": "id",
"type": "INTEGER",
"policyTags": {}
}
]
EOH
}
`, datasetID, tableID)
}

func testAccBigQueryTableBasicSchemaWithEmptyPolicyTagNames(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
schema = <<EOH
[
{
"name": "id",
"type": "INTEGER",
"policyTags": {
"names": []
}
}
]
EOH
}
`, datasetID, tableID)
}

func testAccBigQueryTableTimePartitioning(datasetID, tableID, partitioningType string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
Expand Down

0 comments on commit c964aa6

Please sign in to comment.