Skip to content

Commit

Permalink
Add GCS fields to DiscoveryConfig.yaml.
Browse files Browse the repository at this point in the history
Add tests for GCS terraform
  • Loading branch information
patrickmoy committed Jul 11, 2024
1 parent 67624be commit 1cada78
Show file tree
Hide file tree
Showing 3 changed files with 446 additions and 0 deletions.
128 changes: 128 additions & 0 deletions mmv1/products/dlp/DiscoveryConfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ examples:
primary_resource_id: 'cloud_sql'
test_env_vars:
project: :PROJECT_NAME
- !ruby/object:Provider::Terraform::Examples
name: 'dlp_discovery_config_cloud_storage'
skip_test: true
primary_resource_id: 'cloud_storage'
test_env_vars:
project: :PROJECT_NAME
custom_code: !ruby/object:Provider::Terraform::CustomCode
encoder: templates/terraform/encoders/wrap_object.go.erb
decoder: templates/terraform/decoders/unwrap_resource.go.erb
Expand Down Expand Up @@ -485,6 +491,128 @@ properties:
# The fields below are necessary to include the "secretsDiscoveryTarget" target in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::NestedObject
name: cloudStorageTarget
description: Cloud Storage target for Discovery. The first target to match a bucket will be the one applied.
properties:
- !ruby/object:Api::Type::NestedObject
name: filter
description: The buckets the generation_cadence applies to. The first target with a matching filter will be the one to apply to a bucket.
required: true
properties:
- !ruby/object:Api::Type::NestedObject
name: collection
description: A specific set of buckets for this filter to apply to.
properties:
- !ruby/object:Api::Type::NestedObject
name: includeRegexes
description: A collection of regular expressions to match a file store against.
properties:
- !ruby/object:Api::Type::Array
name: patterns
description: The group of regular expression patterns to match against one or more file stores. Maximum of 100 entries. The sum of all lengths of regular expressions can't exceed 10 KiB.
item_type: !ruby/object:Api::Type::NestedObject
properties:
- !ruby/object:Api::Type::NestedObject
name: cloudStorageRegex
description: Regex for Cloud Storage.
properties:
- !ruby/object:Api::Type::String
name: projectIdRegex
description: For organizations, if unset, will match all projects.
- !ruby/object:Api::Type::String
name: bucketNameRegex
description: 'Regex to test the bucket name against. If empty, all buckets match. Example: "marketing2021" or "(marketing)\d{4}" will both match the bucket gs://marketing2021'
- !ruby/object:Api::Type::NestedObject
name: cloudStorageResourceReference
description: The bucket to scan. Targets including this can only include one target (the target with this bucket). This enables profiling the contents of a single bucket, while the other options allow for easy profiling of many buckets within a project or an organization.
properties:
- !ruby/object:Api::Type::String
name: bucketName
description: The bucket to scan.
- !ruby/object:Api::Type::String
name: projectId
description: If within a project-level config, then this must match the config's project id.
- !ruby/object:Api::Type::NestedObject
name: others
description: Match discovery resources not covered by any other filter.
properties:
[] # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::NestedObject
name: conditions
description: In addition to matching the filter, these conditions must be true before a profile is generated.
properties:
- !ruby/object:Api::Type::String
name: createdAfter
description: File store must have been created after this date. Used to avoid backfilling. A timestamp in RFC3339 UTC "Zulu" format with nanosecond resolution and upto nine fractional digits.
- !ruby/object:Api::Type::String
name: minAge
description: Duration format. Minimum age a file store must have. If set, the value must be 1 hour or greater.
- !ruby/object:Api::Type::NestedObject
name: cloudStorageConditions
description: Cloud Storage conditions.
properties:
- !ruby/object:Api::Type::Array
name: includedObjectAttributes
description: Only objects with the specified attributes will be scanned. If an object has one of the specified attributes but is inside an excluded bucket, it will not be scanned. Defaults to [ALL_SUPPORTED_OBJECTS]. A profile will be created even if no objects match the included_object_attributes.
item_type: !ruby/object:Api::Type::Enum
name: 'undefined'
description: |
This field only has a name and description because of MM
limitations. It should not appear in downstreams.
values:
- :ALL_SUPPORTED_OBJECTS
- :STANDARD
- :NEARLINE
- :COLDLINE
- :ARCHIVE
- :REGIONAL
- :MULTI_REGIONAL
- :DURABLE_REDUCED_AVAILABILITY
- !ruby/object:Api::Type::Array
name: includedBucketAttributes
description: Only objects with the specified attributes will be scanned. Defaults to [ALL_SUPPORTED_BUCKETS] if unset.
item_type: !ruby/object:Api::Type::Enum
name: 'undefined'
description: |
This field only has a name and description because of MM
limitations. It should not appear in downstreams.
values:
- :ALL_SUPPORTED_BUCKETS
- :AUTOCLASS_DISABLED
- :AUTOCLASS_ENABLED
- !ruby/object:Api::Type::NestedObject
name: generationCadence
description: How often and when to update profiles. New buckets that match both the filter and conditions are scanned as quickly as possible depending on system capacity.
properties:
- !ruby/object:Api::Type::Enum
name: refreshFrequency
description: Data changes in Cloud Storage can't trigger reprofiling. If you set this field, profiles are refreshed at this frequency regardless of whether the underlying buckets have changes. Defaults to never.
values:
- :UPDATE_FREQUENCY_NEVER
- :UPDATE_FREQUENCY_DAILY
- :UPDATE_FREQUENCY_MONTHLY
- !ruby/object:Api::Type::NestedObject
name: inspectTemplateModifiedCadence
description: Governs when to update data profiles when the inspection rules defined by the `InspectTemplate` change. If not set, changing the template will not cause a data profile to update.
properties:
- !ruby/object:Api::Type::Enum
name: 'frequency'
description: How frequently data profiles can be updated when the template is modified. Defaults to never.
values:
- :UPDATE_FREQUENCY_NEVER
- :UPDATE_FREQUENCY_DAILY
- :UPDATE_FREQUENCY_MONTHLY
- !ruby/object:Api::Type::NestedObject
name: disabled
description: Disable profiling for buckets that match this filter.
properties:
[] # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled
# The fields below are necessary to include the "disabled" filter in the payload
send_empty_value: true
allow_empty_object: true
- !ruby/object:Api::Type::Array
name: 'errors'
description: Output only. A stream of errors encountered when the config was activated. Repeated errors may result in the config automatically being paused. Output only field. Will return the last 100 errors. Whenever the config is modified this list will be cleared.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
resource "google_data_loss_prevention_discovery_config" "<%= ctx[:primary_resource_id] %>" {
parent = "projects/<%= ctx[:test_env_vars]['project'] %>/locations/us"
location = "us"
status = "RUNNING"

targets {
cloud_storage_target {
filter {
collection {
include_regexes {
patterns {
cloud_storage_regex {
project_id_regex = "foo-project"
bucket_name_regex = "bucket"
}
}
}
}
}
conditions {
created_after = "2023-10-02T15:01:23Z"
min_age = "10800s"
cloud_storage_conditions {
included_object_attributes = ["ALL_SUPPORTED_OBJECTS"]
included_bucket_attributes = ["ALL_SUPPORTED_BUCKETS"]
}
}
generation_cadence {
inspect_template_modified_cadence {
frequency = "UPDATE_FREQUENCY_DAILY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}
}
targets {
cloud_storage_target {
filter {
collection {
include_regexes {
patterns {
cloud_storage_regex {
project_id_regex = "foo-project"
bucket_name_regex = "do-not-scan"
}
}
}
}
}
disabled {}
}
}
targets {
cloud_storage_target {
filter {
others {}
}
generation_cadence {
schema_modified_cadence {
types = ["NEW_COLUMNS"]
frequency = "UPDATE_FREQUENCY_MONTHLY"
}
refresh_frequency = "UPDATE_FREQUENCY_MONTHLY"
}
}

}
inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"]
}

resource "google_data_loss_prevention_inspect_template" "basic" {
parent = "projects/<%= ctx[:test_env_vars]['project'] %>"
description = "My description"
display_name = "display_name"

inspect_config {
info_types {
name = "EMAIL_ADDRESS"
}
}
}
Loading

0 comments on commit 1cada78

Please sign in to comment.