Skip to content

Commit

Permalink
Add cloud_sql_target field in google_data_loss_prevention_discovery_c…
Browse files Browse the repository at this point in the history
…onfig (#10625)
  • Loading branch information
patrickmoy authored May 9, 2024
1 parent e2fa962 commit 87fd42d
Show file tree
Hide file tree
Showing 3 changed files with 347 additions and 1 deletion.
114 changes: 114 additions & 0 deletions mmv1/products/dlp/DiscoveryConfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ examples:
primary_resource_id: 'filter_regexes_and_conditions'
test_env_vars:
project: :PROJECT_NAME
- !ruby/object:Provider::Terraform::Examples
name: 'dlp_discovery_config_cloud_sql'
primary_resource_id: 'cloud_sql'
test_env_vars:
project: :PROJECT_NAME
custom_code: !ruby/object:Provider::Terraform::CustomCode
encoder: templates/terraform/encoders/wrap_object.go.erb
decoder: templates/terraform/decoders/unwrap_resource.go.erb
Expand Down Expand Up @@ -325,6 +330,115 @@ properties:
# The fields below are necessary to include the "disabled" filter in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::NestedObject
name: cloudSqlTarget
description: 'Cloud SQL target for Discovery. The first target to match a table will be the one applied.'
properties:
- !ruby/object:Api::Type::NestedObject
name: filter
description: 'Required. The tables the discovery cadence applies to. The first target with a matching filter will be the one to apply to a table.'
required: true
properties:
- !ruby/object:Api::Type::NestedObject
name: collection
description: 'A specific set of database resources for this filter to apply to.'
properties:
- !ruby/object:Api::Type::NestedObject
name: includeRegexes
description: 'A collection of regular expressions to match a database resource against.'
properties:
- !ruby/object:Api::Type::Array
name: 'patterns'
description: A group of regular expression patterns to match against one or more database resources. Maximum of 100 entries. The sum of all regular expressions' length can't exceed 10 KiB.
item_type: !ruby/object:Api::Type::NestedObject
properties:
- !ruby/object:Api::Type::String
name: projectIdRegex
description: For organizations, if unset, will match all projects. Has no effect for data profile configurations created within a project.
- !ruby/object:Api::Type::String
name: instanceRegex
description: Regex to test the instance name against. If empty, all instances match.
- !ruby/object:Api::Type::String
name: databaseRegex
description: Regex to test the database name against. If empty, all databases match.
- !ruby/object:Api::Type::String
name: databaseResourceNameRegex
description: Regex to test the database resource's name against. An example of a database resource name is a table's name. Other database resource names like view names could be included in the future. If empty, all database resources match.'
- !ruby/object:Api::Type::NestedObject
name: others
description: 'Catch-all. This should always be the last target in the list because anything above it will apply first. Should only appear once in a configuration. If none is specified, a default one will be added automatically.'
properties:
[] # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::NestedObject
name: conditions
description: 'In addition to matching the filter, these conditions must be true before a profile is generated.'
properties:
- !ruby/object:Api::Type::Array
name: databaseEngines
description: Database engines that should be profiled. Optional. Defaults to ALL_SUPPORTED_DATABASE_ENGINES if unspecified.
item_type: !ruby/object:Api::Type::Enum
name: 'undefined'
description: |
This field only has a name and description because of MM
limitations. It should not appear in downstreams.
values:
- :ALL_SUPPORTED_DATABASE_ENGINES
- :MYSQL
- :POSTGRES
- !ruby/object:Api::Type::Array
name: types
description: 'Data profiles will only be generated for the database resource types specified in this field. If not specified, defaults to [DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES].'
item_type: !ruby/object:Api::Type::Enum
name: 'undefined'
description: |
This field only has a name and description because of MM
limitations. It should not appear in downstreams.
values:
- :DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES
- :DATABASE_RESOURCE_TYPE_TABLE
- !ruby/object:Api::Type::NestedObject
name: generationCadence
description: How often and when to update profiles. New tables that match both the filter and conditions are scanned as quickly as possible depending on system capacity.
properties:
- !ruby/object:Api::Type::NestedObject
name: schemaModifiedCadence
description: Governs when to update data profiles when a schema is modified
properties:
- !ruby/object:Api::Type::Array
name: 'types'
description: The types of schema modifications to consider. Defaults to NEW_COLUMNS.
item_type: !ruby/object:Api::Type::Enum
name: 'undefined'
description: |
This field only has a name and description because of MM
limitations. It should not appear in downstreams.
values:
- :NEW_COLUMNS
- :REMOVED_COLUMNS
- !ruby/object:Api::Type::Enum
name: 'frequency'
description: Frequency to regenerate data profiles when the schema is modified. Defaults to monthly.
values:
- :UPDATE_FREQUENCY_NEVER
- :UPDATE_FREQUENCY_DAILY
- :UPDATE_FREQUENCY_MONTHLY
- !ruby/object:Api::Type::Enum
name: 'refreshFrequency'
description: Data changes (non-schema changes) in Cloud SQL tables can't trigger reprofiling. If you set this field, profiles are refreshed at this frequency regardless of whether the underlying tables have changes. Defaults to never.
values:
- :UPDATE_FREQUENCY_NEVER
- :UPDATE_FREQUENCY_DAILY
- :UPDATE_FREQUENCY_MONTHLY
- !ruby/object:Api::Type::NestedObject
name: disabled
description: 'Disable profiling for database resources that match this filter.'
properties:
[] # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled
# The fields below are necessary to include the "disabled" filter in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::Array
name: 'errors'
description: Output only. A stream of errors encountered when the config was activated. Repeated errors may result in the config automatically being paused. Output only field. Will return the last 100 errors. Whenever the config is modified this list will be cleared.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
resource "google_data_loss_prevention_discovery_config" "<%= ctx[:primary_resource_id] %>" {
parent = "projects/<%= ctx[:test_env_vars]['project'] %>/locations/us"
location = "us"
status = "RUNNING"

targets {
cloud_sql_target {
filter {
collection {
include_regexes {
patterns {
project_id_regex = ".*"
instance_regex = ".*"
database_regex = ".*"
database_resource_name_regex = "mytable.*"
}
}
}
}
conditions {
database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"]
types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"]
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS", "REMOVED_COLUMNS"]
frequency = "UPDATE_FREQUENCY_DAILY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}
}
targets {
cloud_sql_target {
filter {
collection {
include_regexes {
patterns {
project_id_regex = ".*"
instance_regex = ".*"
database_regex = "do-not-scan.*"
database_resource_name_regex = ".*"
}
}
}
}
disabled {}
}
}
targets {
cloud_sql_target {
filter {
others {}
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS"]
frequency = "UPDATE_FREQUENCY_MONTHLY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}

}
inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"]
}

resource "google_data_loss_prevention_inspect_template" "basic" {
parent = "projects/<%= ctx[:test_env_vars]['project'] %>"
description = "My description"
display_name = "display_name"

inspect_config {
info_types {
name = "EMAIL_ADDRESS"
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ func TestAccDataLossPreventionDiscoveryConfig_Update(t *testing.T) {
"actions": testAccDataLossPreventionDiscoveryConfig_ActionsUpdate,
"conditions": testAccDataLossPreventionDiscoveryConfig_ConditionsCadenceUpdate,
"filter": testAccDataLossPreventionDiscoveryConfig_FilterUpdate,
"cloud_sql": testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate,
}
for name, tc := range testCases {
// shadow the tc variable into scope so that when
Expand Down Expand Up @@ -213,6 +214,41 @@ func testAccDataLossPreventionDiscoveryConfig_FilterUpdate(t *testing.T) {
})
}

func testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate(t *testing.T) {

context := map[string]interface{}{
"project": envvar.GetTestProjectFromEnv(),
"location": envvar.GetTestRegionFromEnv(),
"random_suffix": acctest.RandString(t, 10),
}

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckDataLossPreventionDiscoveryConfigDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context),
},
{
ResourceName: "google_data_loss_prevention_discovery_config.basic",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"},
},
{
Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context),
},
{
ResourceName: "google_data_loss_prevention_discovery_config.basic",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"},
},
},
})
}

func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStart(context map[string]interface{}) string {
return acctest.Nprintf(`
resource "google_data_loss_prevention_inspect_template" "basic" {
Expand Down Expand Up @@ -326,7 +362,7 @@ resource "google_data_loss_prevention_discovery_config" "basic" {
actions {
export_data {
profile_table {
project_id = "project"
project_id = "%{project}"
dataset_id = "dataset"
table_id = "table"
}
Expand Down Expand Up @@ -584,3 +620,121 @@ resource "google_data_loss_prevention_discovery_config" "basic" {
}
`, context)
}

func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context map[string]interface{}) string {
return acctest.Nprintf(`
resource "google_data_loss_prevention_inspect_template" "basic" {
parent = "projects/%{project}"
description = "Description"
display_name = "Display"
inspect_config {
info_types {
name = "EMAIL_ADDRESS"
}
}
}
resource "google_data_loss_prevention_discovery_config" "basic" {
parent = "projects/%{project}/locations/%{location}"
location = "%{location}"
status = "RUNNING"
targets {
cloud_sql_target {
filter {
collection {
include_regexes {
patterns {
project_id_regex = ".*"
instance_regex = ".*"
database_regex = "do-not-scan.*"
database_resource_name_regex = ".*"
}
}
}
}
conditions {
database_engines = ["MYSQL", "POSTGRES"]
types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"]
}
disabled {}
}
}
targets {
cloud_sql_target {
filter {
others {}
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS"]
frequency = "UPDATE_FREQUENCY_MONTHLY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}
}
inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"]
}
`, context)
}

func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context map[string]interface{}) string {
return acctest.Nprintf(`
resource "google_data_loss_prevention_inspect_template" "basic" {
parent = "projects/%{project}"
description = "Description"
display_name = "Display"
inspect_config {
info_types {
name = "EMAIL_ADDRESS"
}
}
}
resource "google_data_loss_prevention_discovery_config" "basic" {
parent = "projects/%{project}/locations/%{location}"
location = "%{location}"
status = "RUNNING"
targets {
cloud_sql_target {
filter {
collection {
include_regexes {
patterns {
project_id_regex = ".*"
instance_regex = ".*"
database_regex = ".*"
database_resource_name_regex = "mytable.*"
}
}
}
}
conditions {
database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"]
types = ["DATABASE_RESOURCE_TYPE_TABLE"]
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS", "REMOVED_COLUMNS"]
frequency = "UPDATE_FREQUENCY_DAILY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}
}
targets {
cloud_sql_target {
filter {
others {}
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS", "REMOVED_COLUMNS"]
frequency = "UPDATE_FREQUENCY_DAILY"
}
refresh_frequency = "UPDATE_FREQUENCY_DAILY"
}
}
}
inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"]
}
`, context)
}

0 comments on commit 87fd42d

Please sign in to comment.