From deead0c7acfb6ea609be14e69e77a77ddba69efa Mon Sep 17 00:00:00 2001 From: Patrick Moy Date: Tue, 7 May 2024 14:45:03 -0700 Subject: [PATCH 1/2] Add cloudSqlTarget field to DiscoveryConfig.yaml --- mmv1/products/dlp/DiscoveryConfig.yaml | 113 +++++++++++++ .../dlp_discovery_config_cloud_sql.tf.erb | 78 +++++++++ ...a_loss_prevention_discovery_config_test.go | 156 +++++++++++++++++- 3 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb diff --git a/mmv1/products/dlp/DiscoveryConfig.yaml b/mmv1/products/dlp/DiscoveryConfig.yaml index 02b94e47f547..087b11aee309 100644 --- a/mmv1/products/dlp/DiscoveryConfig.yaml +++ b/mmv1/products/dlp/DiscoveryConfig.yaml @@ -63,6 +63,11 @@ examples: primary_resource_id: 'filter_regexes_and_conditions' test_env_vars: project: :PROJECT_NAME + - !ruby/object:Provider::Terraform::Examples + name: 'dlp_discovery_config_cloud_sql' + primary_resource_id: 'cloud_sql' + test_env_vars: + project: :PROJECT_NAME custom_code: !ruby/object:Provider::Terraform::CustomCode encoder: templates/terraform/encoders/wrap_object.go.erb decoder: templates/terraform/decoders/unwrap_resource.go.erb @@ -325,6 +330,114 @@ properties: # The fields below are necessary to include the "disabled" filter in the payload send_empty_value: true allow_empty_object: true + - !ruby/object:Api::Type::NestedObject + name: cloudSqlTarget + description: 'Cloud SQL target for Discovery. The first target to match a table will be the one applied.' + properties: + - !ruby/object:Api::Type::NestedObject + name: filter + description: 'Required. The tables the discovery cadence applies to. The first target with a matching filter will be the one to apply to a table.' + properties: + - !ruby/object:Api::Type::NestedObject + name: collection + description: 'A specific set of database resources for this filter to apply to.' + properties: + - !ruby/object:Api::Type::NestedObject + name: includeRegexes + description: 'A collection of regular expressions to match a database resource against.' + properties: + - !ruby/object:Api::Type::Array + name: 'patterns' + description: A group of regular expression patterns to match against one or more database resources. Maximum of 100 entries. The sum of all regular expressions' length can't exceed 10 KiB. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: projectIdRegex + description: For organizations, if unset, will match all projects. Has no effect for data profile configurations created within a project. + - !ruby/object:Api::Type::String + name: instanceRegex + description: Regex to test the instance name against. If empty, all instances match. + - !ruby/object:Api::Type::String + name: databaseRegex + description: Regex to test the database name against. If empty, all databases match. + - !ruby/object:Api::Type::String + name: databaseResourceNameRegex + description: Regex to test the database resource's name against. An example of a database resource name is a table's name. Other database resource names like view names could be included in the future. If empty, all database resources match.' + - !ruby/object:Api::Type::NestedObject + name: others + description: 'Catch-all. This should always be the last target in the list because anything above it will apply first. Should only appear once in a configuration. If none is specified, a default one will be added automatically.' + properties: + [] # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload + send_empty_value: true + allow_empty_object: true + - !ruby/object:Api::Type::NestedObject + name: conditions + description: 'In addition to matching the filter, these conditions must be true before a profile is generated.' + properties: + - !ruby/object:Api::Type::Array + name: databaseEngines + description: Database engines that should be profiled. Optional. Defaults to ALL_SUPPORTED_DATABASE_ENGINES if unspecified. + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :ALL_SUPPORTED_DATABASE_ENGINES + - :MYSQL + - :POSTGRES + - !ruby/object:Api::Type::Array + name: types + description: 'Data profiles will only be generated for the database resource types specified in this field. If not specified, defaults to [DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES].' + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES + - :DATABASE_RESOURCE_TYPE_TABLE + - !ruby/object:Api::Type::NestedObject + name: generationCadence + description: How often and when to update profiles. New tables that match both the filter and conditions are scanned as quickly as possible depending on system capacity. + properties: + - !ruby/object:Api::Type::NestedObject + name: schemaModifiedCadence + description: Governs when to update data profiles when a schema is modified + properties: + - !ruby/object:Api::Type::Array + name: 'types' + description: The types of schema modifications to consider. Defaults to NEW_COLUMNS. + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :NEW_COLUMNS + - :REMOVED_COLUMNS + - !ruby/object:Api::Type::Enum + name: 'frequency' + description: Frequency to regenerate data profiles when the schema is modified. Defaults to monthly. + values: + - :UPDATE_FREQUENCY_NEVER + - :UPDATE_FREQUENCY_DAILY + - :UPDATE_FREQUENCY_MONTHLY + - !ruby/object:Api::Type::Enum + name: 'refreshFrequency' + description: Data changes (non-schema changes) in Cloud SQL tables can't trigger reprofiling. If you set this field, profiles are refreshed at this frequency regardless of whether the underlying tables have changes. Defaults to never. + values: + - :UPDATE_FREQUENCY_NEVER + - :UPDATE_FREQUENCY_DAILY + - :UPDATE_FREQUENCY_MONTHLY + - !ruby/object:Api::Type::NestedObject + name: disabled + description: 'Disable profiling for database resources that match this filter.' + properties: + [] # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled + # The fields below are necessary to include the "disabled" filter in the payload + send_empty_value: true + allow_empty_object: true - !ruby/object:Api::Type::Array name: 'errors' description: Output only. A stream of errors encountered when the config was activated. Repeated errors may result in the config automatically being paused. Output only field. Will return the last 100 errors. Whenever the config is modified this list will be cleared. diff --git a/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb new file mode 100644 index 000000000000..42e04a1a3093 --- /dev/null +++ b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb @@ -0,0 +1,78 @@ +resource "google_data_loss_prevention_discovery_config" "<%= ctx[:primary_resource_id] %>" { + parent = "projects/<%= ctx[:test_env_vars]['project'] %>/locations/us" + location = "us" + status = "RUNNING" + + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = ".*" + database_resource_name_regex = "mytable.*" + } + } + } + } + conditions { + database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"] + types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"] + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = "do-not-scan.*" + database_resource_name_regex = ".*" + } + } + } + } + disabled {} + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS"] + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} + +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/<%= ctx[:test_env_vars]['project'] %>" + description = "My description" + display_name = "display_name" + + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} \ No newline at end of file diff --git a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go index 09911ce82e68..3bf142f22c3f 100644 --- a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go +++ b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go @@ -15,6 +15,7 @@ func TestAccDataLossPreventionDiscoveryConfig_Update(t *testing.T) { "actions": testAccDataLossPreventionDiscoveryConfig_ActionsUpdate, "conditions": testAccDataLossPreventionDiscoveryConfig_ConditionsCadenceUpdate, "filter": testAccDataLossPreventionDiscoveryConfig_FilterUpdate, + "cloud_sql": testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate, } for name, tc := range testCases { // shadow the tc variable into scope so that when @@ -213,6 +214,41 @@ func testAccDataLossPreventionDiscoveryConfig_FilterUpdate(t *testing.T) { }) } +func testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate(t *testing.T) { + + context := map[string]interface{}{ + "project": envvar.GetTestProjectFromEnv(), + "location": envvar.GetTestRegionFromEnv(), + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataLossPreventionDiscoveryConfigDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"}, + }, + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"}, + }, + }, + }) +} + func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStart(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_data_loss_prevention_inspect_template" "basic" { @@ -326,7 +362,7 @@ resource "google_data_loss_prevention_discovery_config" "basic" { actions { export_data { profile_table { - project_id = "project" + project_id = "%{project}" dataset_id = "dataset" table_id = "table" } @@ -584,3 +620,121 @@ resource "google_data_loss_prevention_discovery_config" "basic" { } `, context) } + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = "do-not-scan.*" + database_resource_name_regex = ".*" + } + } + } + } + conditions { + database_engines = ["MYSQL", "POSTGRES"] + types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"] + } + disabled {} + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS"] + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +} + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = ".*" + database_resource_name_regex = "mytable.*" + } + } + } + } + conditions { + database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"] + types = ["DATABASE_RESOURCE_TYPE_TABLE"] + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_DAILY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +} From 4d7f34cc06310e21d45ff31711387e362713f212 Mon Sep 17 00:00:00 2001 From: Patrick Moy Date: Wed, 8 May 2024 14:01:09 -0700 Subject: [PATCH 2/2] Fix spacing issues and add required tag to cloud_sql_target.filter --- mmv1/products/dlp/DiscoveryConfig.yaml | 1 + .../dlp_discovery_config_cloud_sql.tf.erb | 14 ++--- ...a_loss_prevention_discovery_config_test.go | 56 +++++++++---------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/mmv1/products/dlp/DiscoveryConfig.yaml b/mmv1/products/dlp/DiscoveryConfig.yaml index 087b11aee309..137a33fdba9e 100644 --- a/mmv1/products/dlp/DiscoveryConfig.yaml +++ b/mmv1/products/dlp/DiscoveryConfig.yaml @@ -337,6 +337,7 @@ properties: - !ruby/object:Api::Type::NestedObject name: filter description: 'Required. The tables the discovery cadence applies to. The first target with a matching filter will be the one to apply to a table.' + required: true properties: - !ruby/object:Api::Type::NestedObject name: collection diff --git a/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb index 42e04a1a3093..03b2fa82ec36 100644 --- a/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb +++ b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb @@ -66,13 +66,13 @@ resource "google_data_loss_prevention_discovery_config" "<%= ctx[:primary_resour } resource "google_data_loss_prevention_inspect_template" "basic" { - parent = "projects/<%= ctx[:test_env_vars]['project'] %>" - description = "My description" - display_name = "display_name" + parent = "projects/<%= ctx[:test_env_vars]['project'] %>" + description = "My description" + display_name = "display_name" - inspect_config { - info_types { - name = "EMAIL_ADDRESS" - } + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } } } \ No newline at end of file diff --git a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go index 3bf142f22c3f..58c38d206033 100644 --- a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go +++ b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go @@ -624,19 +624,19 @@ resource "google_data_loss_prevention_discovery_config" "basic" { func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_data_loss_prevention_inspect_template" "basic" { - parent = "projects/%{project}" - description = "Description" - display_name = "Display" - inspect_config { - info_types { - name = "EMAIL_ADDRESS" - } - } + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } } resource "google_data_loss_prevention_discovery_config" "basic" { - parent = "projects/%{project}/locations/%{location}" - location = "%{location}" - status = "RUNNING" + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" targets { cloud_sql_target { filter { @@ -651,7 +651,7 @@ resource "google_data_loss_prevention_discovery_config" "basic" { } } } - conditions { + conditions { database_engines = ["MYSQL", "POSTGRES"] types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"] } @@ -663,7 +663,7 @@ resource "google_data_loss_prevention_discovery_config" "basic" { filter { others {} } - generation_cadence { + generation_cadence { schema_modified_cadence { types = ["NEW_COLUMNS"] frequency = "UPDATE_FREQUENCY_MONTHLY" @@ -680,20 +680,20 @@ resource "google_data_loss_prevention_discovery_config" "basic" { func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_data_loss_prevention_inspect_template" "basic" { - parent = "projects/%{project}" - description = "Description" - display_name = "Display" - inspect_config { - info_types { - name = "EMAIL_ADDRESS" - } - } + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } } resource "google_data_loss_prevention_discovery_config" "basic" { - parent = "projects/%{project}/locations/%{location}" - location = "%{location}" - status = "RUNNING" - targets { + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" + targets { cloud_sql_target { filter { collection { @@ -719,13 +719,13 @@ resource "google_data_loss_prevention_discovery_config" "basic" { refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" } } - } - targets { + } + targets { cloud_sql_target { filter { others {} } - generation_cadence { + generation_cadence { schema_modified_cadence { types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] frequency = "UPDATE_FREQUENCY_DAILY"