Skip to content

Commit

Permalink
feat(bigquery): add table resource tags support (#9084)
Browse files Browse the repository at this point in the history
This PR adds support for resource tags on BigQuery tables.

Due to the nature of the feature, testing necessitates special provisioning with resourcemanager to establish a parent with the proper keys and values defined, which can then be bound to the table.

While implementing, testing was done in a personal test project, with the following commands to establish the necessary tags/values:
```
gcloud resource-manager tags keys create test_tag_key --parent=projects/shollyman-testing

gcloud resource-manager tags values create COFFEE --parent=tagKeys/281483438148747
gcloud resource-manager tags values create TEA --parent=tagKeys/281483438148747
gcloud resource-manager tags values create WATER --parent=tagKeys/281483438148747
```

This integration test was used to smoke test that the feature works:
```
func TestIntegration_TableResourceTags(t *testing.T) {
	if client == nil {
		t.Skip("Integration tests skipped")
	}

	testKey := "shollyman-testing/test_tag_key"

	ctx := context.Background()
	table := dataset.Table(tableIDs.New())
	resourceTags := map[string]string{
		testKey: "COFFEE",
	}
	if err := table.Create(context.Background(), &TableMetadata{
		Schema:       schema,
		ResourceTags: resourceTags,
	}); err != nil {
		t.Fatalf("table.Create: %v", err)
	}
	defer table.Delete(ctx)
	md, err := table.Metadata(ctx)
	if err != nil {
		t.Fatalf("table.Metadata: %v", err)
	}

	var found bool
	for k, v := range md.ResourceTags {
		if k == testKey && v == "COFFEE" {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("tag key/value not found")
	}

	updatedTags := map[string]string{
		testKey: "COFFEE",
	}
	// Update table DefaultCollation to case-sensitive
	updated, err := table.Update(ctx, TableMetadataToUpdate{
		ResourceTags: updatedTags,
	}, md.ETag)
	if err != nil {
		t.Fatalf("table.Update: %v", err)
	}

	found = false
	for k, v := range updated.ResourceTags {
		if k == testKey && v == "COFFEE" {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("tag key/value not found")
	}

}
```

I've removed the integration test for the time being after verification, given the complexity of the setup outside of the BQ service.
  • Loading branch information
shollyman authored Dec 14, 2023
1 parent 365c5fc commit 3569cc2
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 0 deletions.
37 changes: 37 additions & 0 deletions bigquery/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,15 @@ type TableMetadata struct {
// TableConstraints contains table primary and foreign keys constraints.
// Present only if the table has primary or foreign keys.
TableConstraints *TableConstraints

// The tags associated with this table. Tag
// keys are globally unique. See additional information on tags
// (https://cloud.google.com/iam/docs/tags-access-control#definitions).
// An object containing a list of "key": value pairs. The key is the
// namespaced friendly name of the tag key, e.g. "12345/environment"
// where 12345 is parent id. The value is the friendly short name of the
// tag value, e.g. "production".
ResourceTags map[string]string
}

// TableConstraints defines the primary key and foreign key of a table.
Expand Down Expand Up @@ -788,6 +797,12 @@ func (tm *TableMetadata) toBQ() (*bq.Table, error) {
}
}
}
if tm.ResourceTags != nil {
t.ResourceTags = make(map[string]string)
for k, v := range tm.ResourceTags {
t.ResourceTags[k] = v
}
}
return t, nil
}

Expand Down Expand Up @@ -907,6 +922,12 @@ func bqToTableMetadata(t *bq.Table, c *Client) (*TableMetadata, error) {
ForeignKeys: bqToForeignKeys(t.TableConstraints, c),
}
}
if t.ResourceTags != nil {
md.ResourceTags = make(map[string]string)
for k, v := range t.ResourceTags {
md.ResourceTags[k] = v
}
}
return md, nil
}

Expand Down Expand Up @@ -1081,6 +1102,13 @@ func (tm *TableMetadataToUpdate) toBQ() (*bq.Table, error) {
t.TableConstraints.ForceSendFields = append(t.TableConstraints.ForceSendFields, "ForeignKeys")
}
}
if tm.ResourceTags != nil {
t.ResourceTags = make(map[string]string)
for k, v := range tm.ResourceTags {
t.ResourceTags[k] = v
}
forceSend("ResourceTags")
}
labels, forces, nulls := tm.update()
t.Labels = labels
t.ForceSendFields = append(t.ForceSendFields, forces...)
Expand Down Expand Up @@ -1162,6 +1190,15 @@ type TableMetadataToUpdate struct {
// such as primary and foreign keys.
TableConstraints *TableConstraints

// The tags associated with this table. Tag
// keys are globally unique. See additional information on tags
// (https://cloud.google.com/iam/docs/tags-access-control#definitions).
// An object containing a list of "key": value pairs. The key is the
// namespaced friendly name of the tag key, e.g. "12345/environment"
// where 12345 is parent id. The value is the friendly short name of the
// tag value, e.g. "production".
ResourceTags map[string]string

labelUpdater
}

Expand Down
31 changes: 31 additions & 0 deletions bigquery/table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ func TestBQToTableMetadata(t *testing.T) {
},
},
},
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
},
&TableMetadata{
Description: "desc",
Expand Down Expand Up @@ -155,6 +159,10 @@ func TestBQToTableMetadata(t *testing.T) {
},
},
},
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
},
},
} {
Expand Down Expand Up @@ -188,6 +196,10 @@ func TestTableMetadataToBQ(t *testing.T) {
Labels: map[string]string{"a": "b"},
ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
},
&bq.Table{
FriendlyName: "n",
Expand All @@ -201,6 +213,10 @@ func TestTableMetadataToBQ(t *testing.T) {
Labels: map[string]string{"a": "b"},
ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
},
},
{
Expand Down Expand Up @@ -511,6 +527,21 @@ func TestTableMetadataToUpdateToBQ(t *testing.T) {
},
},
},
{
tm: TableMetadataToUpdate{
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
},
want: &bq.Table{
ResourceTags: map[string]string{
"key1": "val1",
"key2": "val2",
},
ForceSendFields: []string{"ResourceTags"},
},
},
} {
got, _ := test.tm.toBQ()
if !testutil.Equal(got, test.want) {
Expand Down

0 comments on commit 3569cc2

Please sign in to comment.