Skip to content

Commit

Permalink
BigQuery Authorized datasets (GoogleCloudPlatform#5700)
Browse files Browse the repository at this point in the history
* Add authorized views to bigquery_dataset

* Add authorized view to dataset_access

* Fix test file

* Add required to nested dataset field
  • Loading branch information
slevenick authored and lcaggio committed Mar 16, 2022
1 parent 934481b commit f2871b0
Show file tree
Hide file tree
Showing 7 changed files with 272 additions and 0 deletions.
67 changes: 67 additions & 0 deletions mmv1/products/bigquery/api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,32 @@ objects:
A-Z), numbers (0-9), or underscores (_). The maximum length
is 1,024 characters.
required: true
- !ruby/object:Api::Type::NestedObject
name: 'dataset'
description: |
Grants all resources of particular types in a particular dataset read access to the current dataset.
properties:
- !ruby/object:Api::Type::NestedObject
name: 'dataset'
required: true
description: |
The dataset this entry applies to
properties:
- !ruby/object:Api::Type::String
name: 'datasetId'
description: The ID of the dataset containing this table.
required: true
- !ruby/object:Api::Type::String
name: 'projectId'
description: The ID of the project containing this table.
required: true
- !ruby/object:Api::Type::Array
name: 'targetTypes'
description: |
Which resources in the dataset this entry applies to. Currently, only views are supported,
but additional target types may be added in the future. Possible values: VIEWS
item_type: Api::Type::String
required: true
- !ruby/object:Api::Type::Integer
name: 'creationTime'
output: true
Expand Down Expand Up @@ -248,6 +274,7 @@ objects:
- specialGroup
- iamMember
- view
- dataset
description: |
Gives dataset access for a single entity. This resource is intended to be used in cases where
it is not possible to compile a full list of access blocks to include in a
Expand Down Expand Up @@ -289,6 +316,7 @@ objects:
- special_group
- iam_member
- view
- dataset
- !ruby/object:Api::Type::String
name: 'groupByEmail'
description: An email address of a Google Group to grant access to.
Expand All @@ -299,6 +327,7 @@ objects:
- special_group
- iam_member
- view
- dataset
- !ruby/object:Api::Type::String
name: 'domain'
description: |
Expand All @@ -311,6 +340,7 @@ objects:
- special_group
- iam_member
- view
- dataset
- !ruby/object:Api::Type::String
name: 'specialGroup'
description: |
Expand All @@ -334,6 +364,7 @@ objects:
- special_group
- iam_member
- view
- dataset
- !ruby/object:Api::Type::String
name: 'iamMember'
description: |
Expand All @@ -346,6 +377,7 @@ objects:
- special_group
- iam_member
- view
- dataset
- !ruby/object:Api::Type::NestedObject
name: 'view'
description: |
Expand All @@ -361,6 +393,7 @@ objects:
- special_group
- iam_member
- view
- dataset
properties:
- !ruby/object:Api::Type::String
name: 'datasetId'
Expand All @@ -377,6 +410,40 @@ objects:
A-Z), numbers (0-9), or underscores (_). The maximum length
is 1,024 characters.
required: true
- !ruby/object:Api::Type::NestedObject
name: 'dataset'
description: |
Grants all resources of particular types in a particular dataset read access to the current dataset.
exactly_one_of:
- user_by_email
- group_by_email
- domain
- special_group
- iam_member
- view
- dataset
properties:
- !ruby/object:Api::Type::NestedObject
name: 'dataset'
required: true
description: |
The dataset this entry applies to
properties:
- !ruby/object:Api::Type::String
name: 'datasetId'
description: The ID of the dataset containing this table.
required: true
- !ruby/object:Api::Type::String
name: 'projectId'
description: The ID of the project containing this table.
required: true
- !ruby/object:Api::Type::Array
name: 'targetTypes'
description: |
Which resources in the dataset this entry applies to. Currently, only views are supported,
but additional target types may be added in the future. Possible values: VIEWS
item_type: Api::Type::String
required: true
- !ruby/object:Api::Resource
name: 'Job'
kind: 'bigquery#job'
Expand Down
14 changes: 14 additions & 0 deletions mmv1/products/bigquery/terraform.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ overrides: !ruby/object:Overrides::ResourceOverrides
dataset_id: "example_dataset"
key_name: "example-key"
keyring_name: "example-keyring"
- !ruby/object:Provider::Terraform::Examples
name: "bigquery_dataset_authorized_dataset"
primary_resource_id: "dataset"
vars:
private: "private"
public: "public"
account_name: "bqowner"
virtual_fields:
- !ruby/object:Api::Type::Boolean
name: 'delete_contents_on_destroy'
Expand Down Expand Up @@ -95,6 +102,13 @@ overrides: !ruby/object:Overrides::ResourceOverrides
dataset_id: "example_dataset"
dataset_id2: "example_dataset2"
table_id: "example_table"
- !ruby/object:Provider::Terraform::Examples
name: "bigquery_dataset_access_authorized_dataset"
skip_test: true # not importable
primary_resource_id: "access"
vars:
private: "private"
public: "public"
properties:
datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
ignore_read: true
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
resource "google_bigquery_dataset_access" "<%= ctx[:primary_resource_id] %>" {
dataset_id = google_bigquery_dataset.private.dataset_id
dataset {
dataset{
project_id = google_bigquery_dataset.public.project
dataset_id = google_bigquery_dataset.public.dataset_id
}
target_types = ["VIEWS"]
}
}

resource "google_bigquery_dataset" "private" {
dataset_id = "<%= ctx[:vars]['private'] %>"
}

resource "google_bigquery_dataset" "public" {
dataset_id = "<%= ctx[:vars]['public'] %>"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
resource "google_bigquery_dataset" "public" {
dataset_id = "<%= ctx[:vars]['public'] %>"
friendly_name = "test"
description = "This dataset is public"
location = "EU"
default_table_expiration_ms = 3600000

labels = {
env = "default"
}

access {
role = "OWNER"
user_by_email = google_service_account.bqowner.email
}

access {
role = "READER"
domain = "hashicorp.com"
}
}

resource "google_bigquery_dataset" "<%= ctx[:primary_resource_id] %>" {
dataset_id = "<%= ctx[:vars]['private'] %>"
friendly_name = "test"
description = "This dataset is private"
location = "EU"
default_table_expiration_ms = 3600000

labels = {
env = "default"
}

access {
role = "OWNER"
user_by_email = google_service_account.bqowner.email
}

access {
role = "READER"
domain = "hashicorp.com"
}

access {
dataset {
dataset {
project_id = google_bigquery_dataset.public.project
dataset_id = google_bigquery_dataset.public.dataset_id
}
target_types = ["VIEWS"]
}
}
}

resource "google_service_account" "bqowner" {
account_id = "<%= ctx[:vars]['account_name'] %>"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
resource "google_bigquery_dataset" "public" {
dataset_id = "<%= ctx[:vars]['grant_dataset_id'] %>"
friendly_name = "test"
description = "This dataset is public"
location = "EU"
default_table_expiration_ms = 3600000

labels = {
env = "default"
}

access {
role = "OWNER"
user_by_email = google_service_account.bqowner.email
}

access {
role = "READER"
domain = "hashicorp.com"
}
}

resource "google_bigquery_dataset" "<%= ctx[:primary_resource_id] %>" {
dataset_id = "<%= ctx[:vars]['dataset_id'] %>"
friendly_name = "test"
description = "This dataset is private"
location = "EU"
default_table_expiration_ms = 3600000

labels = {
env = "default"
}

access {
role = "OWNER"
user_by_email = google_service_account.bqowner.email
}

access {
role = "READER"
domain = "hashicorp.com"
}

access {
dataset {
dataset {
project_id = google_bigquery_dataset.to_authorize.project
dataset_id = google_bigquery_dataset.to_authorize.dataset_id
}
target_types = ["VIEWS"]
}
}
}

resource "google_service_account" "bqowner" {
account_id = "<%= ctx[:vars]['account_name'] %>"
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,38 @@ func TestAccBigQueryDatasetAccess_view(t *testing.T) {
})
}

func TestAccBigQueryDatasetAccess_authorizedDataset(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", randString(t, 10))
datasetID2 := fmt.Sprintf("tf_test_%s", randString(t, 10))

expected := map[string]interface{}{
"dataset": map[string]interface{}{
"dataset": map[string]interface{}{
"projectId": getTestProjectFromEnv(),
"datasetId": datasetID2,
},
"targetTypes": []interface{}{"VIEWS"},
},
}

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccBigQueryDatasetAccess_authorizedDataset(datasetID, datasetID2),
Check: testAccCheckBigQueryDatasetAccessPresent(t, "google_bigquery_dataset.private", expected),
},
{
Config: testAccBigQueryDatasetAccess_destroy(datasetID, "private"),
Check: testAccCheckBigQueryDatasetAccessAbsent(t, "google_bigquery_dataset.private", expected),
},
},
})
}

func TestAccBigQueryDatasetAccess_multiple(t *testing.T) {
// Multiple fine-grained resources
skipIfVcr(t)
Expand Down Expand Up @@ -303,6 +335,29 @@ resource "google_bigquery_table" "public" {
`, datasetID, datasetID2, tableID, "SELECT state FROM `lookerdata.cdc.project_tycho_reports`")
}

func testAccBigQueryDatasetAccess_authorizedDataset(datasetID, datasetID2 string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset_access" "access" {
dataset_id = google_bigquery_dataset.private.dataset_id
dataset {
dataset{
project_id = google_bigquery_dataset.public.project
dataset_id = google_bigquery_dataset.public.dataset_id
}
target_types = ["VIEWS"]
}
}
resource "google_bigquery_dataset" "private" {
dataset_id = "%s"
}
resource "google_bigquery_dataset" "public" {
dataset_id = "%s"
}
`, datasetID, datasetID2)
}

func testAccBigQueryDatasetAccess_multiple(datasetID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset_access" "access" {
Expand Down
4 changes: 4 additions & 0 deletions mmv1/third_party/terraform/utils/iam_bigquery_dataset.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,10 @@ func accessToIamMember(access map[string]interface{}) (string, error) {
// view does not map to an IAM member, use access instead
return "", fmt.Errorf("Failed to convert BigQuery Dataset access to IAM member. To use views with a dataset, please use dataset_access")
}
if _, ok := access["dataset"]; ok {
// dataset does not map to an IAM member, use access instead
return "", fmt.Errorf("Failed to convert BigQuery Dataset access to IAM member. To use views with a dataset, please use dataset_access")
}
if member, ok := access["userByEmail"]; ok {
// service accounts have "gservice" in their email. This is best guess due to lost information
if strings.Contains(member.(string), "gserviceaccount") {
Expand Down

0 comments on commit f2871b0

Please sign in to comment.