diff --git a/mmv1/products/dlp/DiscoveryConfig.yaml b/mmv1/products/dlp/DiscoveryConfig.yaml index 02b94e47f547..137a33fdba9e 100644 --- a/mmv1/products/dlp/DiscoveryConfig.yaml +++ b/mmv1/products/dlp/DiscoveryConfig.yaml @@ -63,6 +63,11 @@ examples: primary_resource_id: 'filter_regexes_and_conditions' test_env_vars: project: :PROJECT_NAME + - !ruby/object:Provider::Terraform::Examples + name: 'dlp_discovery_config_cloud_sql' + primary_resource_id: 'cloud_sql' + test_env_vars: + project: :PROJECT_NAME custom_code: !ruby/object:Provider::Terraform::CustomCode encoder: templates/terraform/encoders/wrap_object.go.erb decoder: templates/terraform/decoders/unwrap_resource.go.erb @@ -325,6 +330,115 @@ properties: # The fields below are necessary to include the "disabled" filter in the payload send_empty_value: true allow_empty_object: true + - !ruby/object:Api::Type::NestedObject + name: cloudSqlTarget + description: 'Cloud SQL target for Discovery. The first target to match a table will be the one applied.' + properties: + - !ruby/object:Api::Type::NestedObject + name: filter + description: 'Required. The tables the discovery cadence applies to. The first target with a matching filter will be the one to apply to a table.' + required: true + properties: + - !ruby/object:Api::Type::NestedObject + name: collection + description: 'A specific set of database resources for this filter to apply to.' + properties: + - !ruby/object:Api::Type::NestedObject + name: includeRegexes + description: 'A collection of regular expressions to match a database resource against.' + properties: + - !ruby/object:Api::Type::Array + name: 'patterns' + description: A group of regular expression patterns to match against one or more database resources. Maximum of 100 entries. The sum of all regular expressions' length can't exceed 10 KiB. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: projectIdRegex + description: For organizations, if unset, will match all projects. Has no effect for data profile configurations created within a project. + - !ruby/object:Api::Type::String + name: instanceRegex + description: Regex to test the instance name against. If empty, all instances match. + - !ruby/object:Api::Type::String + name: databaseRegex + description: Regex to test the database name against. If empty, all databases match. + - !ruby/object:Api::Type::String + name: databaseResourceNameRegex + description: Regex to test the database resource's name against. An example of a database resource name is a table's name. Other database resource names like view names could be included in the future. If empty, all database resources match.' + - !ruby/object:Api::Type::NestedObject + name: others + description: 'Catch-all. This should always be the last target in the list because anything above it will apply first. Should only appear once in a configuration. If none is specified, a default one will be added automatically.' + properties: + [] # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload + send_empty_value: true + allow_empty_object: true + - !ruby/object:Api::Type::NestedObject + name: conditions + description: 'In addition to matching the filter, these conditions must be true before a profile is generated.' + properties: + - !ruby/object:Api::Type::Array + name: databaseEngines + description: Database engines that should be profiled. Optional. Defaults to ALL_SUPPORTED_DATABASE_ENGINES if unspecified. + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :ALL_SUPPORTED_DATABASE_ENGINES + - :MYSQL + - :POSTGRES + - !ruby/object:Api::Type::Array + name: types + description: 'Data profiles will only be generated for the database resource types specified in this field. If not specified, defaults to [DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES].' + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES + - :DATABASE_RESOURCE_TYPE_TABLE + - !ruby/object:Api::Type::NestedObject + name: generationCadence + description: How often and when to update profiles. New tables that match both the filter and conditions are scanned as quickly as possible depending on system capacity. + properties: + - !ruby/object:Api::Type::NestedObject + name: schemaModifiedCadence + description: Governs when to update data profiles when a schema is modified + properties: + - !ruby/object:Api::Type::Array + name: 'types' + description: The types of schema modifications to consider. Defaults to NEW_COLUMNS. + item_type: !ruby/object:Api::Type::Enum + name: 'undefined' + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + values: + - :NEW_COLUMNS + - :REMOVED_COLUMNS + - !ruby/object:Api::Type::Enum + name: 'frequency' + description: Frequency to regenerate data profiles when the schema is modified. Defaults to monthly. + values: + - :UPDATE_FREQUENCY_NEVER + - :UPDATE_FREQUENCY_DAILY + - :UPDATE_FREQUENCY_MONTHLY + - !ruby/object:Api::Type::Enum + name: 'refreshFrequency' + description: Data changes (non-schema changes) in Cloud SQL tables can't trigger reprofiling. If you set this field, profiles are refreshed at this frequency regardless of whether the underlying tables have changes. Defaults to never. + values: + - :UPDATE_FREQUENCY_NEVER + - :UPDATE_FREQUENCY_DAILY + - :UPDATE_FREQUENCY_MONTHLY + - !ruby/object:Api::Type::NestedObject + name: disabled + description: 'Disable profiling for database resources that match this filter.' + properties: + [] # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled + # The fields below are necessary to include the "disabled" filter in the payload + send_empty_value: true + allow_empty_object: true - !ruby/object:Api::Type::Array name: 'errors' description: Output only. A stream of errors encountered when the config was activated. Repeated errors may result in the config automatically being paused. Output only field. Will return the last 100 errors. Whenever the config is modified this list will be cleared. diff --git a/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb new file mode 100644 index 000000000000..03b2fa82ec36 --- /dev/null +++ b/mmv1/templates/terraform/examples/dlp_discovery_config_cloud_sql.tf.erb @@ -0,0 +1,78 @@ +resource "google_data_loss_prevention_discovery_config" "<%= ctx[:primary_resource_id] %>" { + parent = "projects/<%= ctx[:test_env_vars]['project'] %>/locations/us" + location = "us" + status = "RUNNING" + + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = ".*" + database_resource_name_regex = "mytable.*" + } + } + } + } + conditions { + database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"] + types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"] + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = "do-not-scan.*" + database_resource_name_regex = ".*" + } + } + } + } + disabled {} + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS"] + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} + +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/<%= ctx[:test_env_vars]['project'] %>" + description = "My description" + display_name = "display_name" + + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} \ No newline at end of file diff --git a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go index 09911ce82e68..58c38d206033 100644 --- a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go +++ b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go @@ -15,6 +15,7 @@ func TestAccDataLossPreventionDiscoveryConfig_Update(t *testing.T) { "actions": testAccDataLossPreventionDiscoveryConfig_ActionsUpdate, "conditions": testAccDataLossPreventionDiscoveryConfig_ConditionsCadenceUpdate, "filter": testAccDataLossPreventionDiscoveryConfig_FilterUpdate, + "cloud_sql": testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate, } for name, tc := range testCases { // shadow the tc variable into scope so that when @@ -213,6 +214,41 @@ func testAccDataLossPreventionDiscoveryConfig_FilterUpdate(t *testing.T) { }) } +func testAccDataLossPreventionDiscoveryConfig_CloudSqlUpdate(t *testing.T) { + + context := map[string]interface{}{ + "project": envvar.GetTestProjectFromEnv(), + "location": envvar.GetTestRegionFromEnv(), + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataLossPreventionDiscoveryConfigDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"}, + }, + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time"}, + }, + }, + }) +} + func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStart(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_data_loss_prevention_inspect_template" "basic" { @@ -326,7 +362,7 @@ resource "google_data_loss_prevention_discovery_config" "basic" { actions { export_data { profile_table { - project_id = "project" + project_id = "%{project}" dataset_id = "dataset" table_id = "table" } @@ -584,3 +620,121 @@ resource "google_data_loss_prevention_discovery_config" "basic" { } `, context) } + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartCloudSql(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = "do-not-scan.*" + database_resource_name_regex = ".*" + } + } + } + } + conditions { + database_engines = ["MYSQL", "POSTGRES"] + types = ["DATABASE_RESOURCE_TYPE_ALL_SUPPORTED_TYPES"] + } + disabled {} + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS"] + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +} + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateCloudSql(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "projects/%{project}/locations/%{location}" + location = "%{location}" + status = "RUNNING" + targets { + cloud_sql_target { + filter { + collection { + include_regexes { + patterns { + project_id_regex = ".*" + instance_regex = ".*" + database_regex = ".*" + database_resource_name_regex = "mytable.*" + } + } + } + } + conditions { + database_engines = ["ALL_SUPPORTED_DATABASE_ENGINES"] + types = ["DATABASE_RESOURCE_TYPE_TABLE"] + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + targets { + cloud_sql_target { + filter { + others {} + } + generation_cadence { + schema_modified_cadence { + types = ["NEW_COLUMNS", "REMOVED_COLUMNS"] + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_DAILY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +}