diff --git a/dlp/snippets/README.md b/dlp/snippets/README.md new file mode 100644 index 00000000000..dffa29460d6 --- /dev/null +++ b/dlp/snippets/README.md @@ -0,0 +1,59 @@ +# Cloud Data Loss Prevention (DLP) API Samples + + +Open in Cloud Shell + +The [Data Loss Prevention API](https://cloud.google.com/dlp/docs/) provides programmatic access to +a powerful detection engine for personally identifiable information and other privacy-sensitive data + in unstructured data streams. + +## Setup +- A Google Cloud project with billing enabled +- [Enable](https://console.cloud.google.com/launcher/details/google/dlp.googleapis.com) the DLP API. +- [Create a service account](https://cloud.google.com/docs/authentication/getting-started) +and set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to the downloaded credentials file. + +## Running + +To run a specific sample, edit any variables under the `TODO(developer):` in the +function at the top of each sample, and then execute the function as convenient. + +For example, if using the command line you might use the following (replacing +`` with the name of the sample): +```bash +mvn exec:java -Dexec.mainClass="dlp.snippets." +``` + + +## Testing + +### Setup +- Ensure that `GOOGLE_APPLICATION_CREDENTIALS` points to authorized service account credentials file. +- Set the `DLP_DEID_WRAPPED_KEY` environment variable to an AES-256 key encrypted ('wrapped') [with a Cloud Key Management Service (KMS) key](https://cloud.google.com/kms/docs/encrypt-decrypt). +- Set the `DLP_DEID_KEY_NAME` environment variable to the path-name of the Cloud KMS key you wrapped `DLP_DEID_WRAPPED_KEY` with. +- [Create a Google Cloud Storage bucket](https://console.cloud.google.com/storage) and upload [test.txt](src/test/resources/test.txt). + - Set the `GCS_PATH` environment variable to point to the path for the bucket. +- Copy and paste the data below into a CSV file and [create a BigQuery table](https://cloud.google.com/bigquery/docs/loading-data-local) from the file: + ```$xslt + Name,TelephoneNumber,Mystery,Age,Gender + James,(567) 890-1234,8291 3627 8250 1234,19,Male + Gandalf,(223) 456-7890,4231 5555 6781 9876,27,Male + Dumbledore,(313) 337-1337,6291 8765 1095 7629,27,Male + Joe,(452) 223-1234,3782 2288 1166 3030,35,Male + Marie,(452) 223-1234,8291 3627 8250 1234,35,Female + Carrie,(567) 890-1234,2253 5218 4251 4526,35,Female + ``` + - Set the `BIGQUERY_DATASET` and `BIGQUERY_TABLE` environment values. +- [Create a Google Cloud Pub/Sub](https://console.cloud.google.com/datastore) topic and and a subscription that is subscribed to the topic. + - Set the `PUB_SUB_TOPIC` and `PUB_SUB_SUBSCRIPTION` environment variables to the corresponding values. +- [Create a Google Cloud Datastore](https://console.cloud.google.com/datastore) kind and add an entity with properties: + - `property1` : john@doe.com + - `property2` : 343-343-3435 +- Update the Datastore kind in [InspectTests.java](src/test/java/dlp/snippets/InspectTests.java). + + +### Run +Run all tests: +``` + mvn clean verify +``` diff --git a/dlp/snippets/pom.xml b/dlp/snippets/pom.xml new file mode 100644 index 00000000000..c635bdf2bfb --- /dev/null +++ b/dlp/snippets/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + com.example.dlp + dlp-snippets + jar + Google Cloud Data Loss Prevention Snippets + https://github.com/GoogleCloudPlatform/java-docs-samples/tree/main/dlp + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + com.google.cloud + google-cloud-dlp + 3.12.0 + + + + com.google.cloud + google-cloud-pubsub + 1.120.24 + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + + + diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableBucketing.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableBucketing.java new file mode 100644 index 00000000000..bf3d70823d4 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableBucketing.java @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_bucketing] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.FixedSizeBucketingConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; + +public class DeIdentifyTableBucketing { + + public static void deIdentifyTableBucketing() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + + deIdentifyTableBucketing(projectId, tableToDeIdentify); + } + + public static Table deIdentifyTableBucketing(String projectId, Table tableToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify how the content should be de-identified. + FixedSizeBucketingConfig fixedSizeBucketingConfig = + FixedSizeBucketingConfig.newBuilder() + .setBucketSize(10) + .setLowerBound(Value.newBuilder().setIntegerValue(0).build()) + .setUpperBound(Value.newBuilder().setIntegerValue(100).build()) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setFixedSizeBucketingConfig(fixedSizeBucketingConfig) + .build(); + + // Specify field to be encrypted. + FieldId fieldId = FieldId.newBuilder().setName("HAPPINESS SCORE").build(); + + // Associate the encryption with the specified field. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addFields(fieldId) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println("Table after de-identification: " + response.getItem().getTable()); + + return response.getItem().getTable(); + } + } +} +// [END dlp_deidentify_table_bucketing] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionInfoTypes.java new file mode 100644 index 00000000000..4fe50794f3b --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionInfoTypes.java @@ -0,0 +1,173 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_condition_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordCondition; +import com.google.privacy.dlp.v2.RecordCondition.Condition; +import com.google.privacy.dlp.v2.RecordCondition.Conditions; +import com.google.privacy.dlp.v2.RecordCondition.Expressions; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.RelationalOperator; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class DeIdentifyTableConditionInfoTypes { + + public static void deIdentifyTableConditionInfoTypes() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "Charles Dickens name was a curse invented by Shakespeare.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in Jane Austen's novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) + .build()) + .build(); + + deIdentifyTableConditionInfoTypes(projectId, tableToDeIdentify); + } + + public static Table deIdentifyTableConditionInfoTypes(String projectId, Table tableToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify how the content should be de-identified. + // Select type of info to be replaced. + InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build(); + // Specify that findings should be replaced with corresponding info type name. + ReplaceWithInfoTypeConfig replaceWithInfoTypeConfig = + ReplaceWithInfoTypeConfig.getDefaultInstance(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(replaceWithInfoTypeConfig) + .build(); + // Associate info type with the replacement strategy + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + InfoTypeTransformations infoTypeTransformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + // Specify fields to be de-identified. + List fieldIds = + Stream.of("PATIENT", "FACTOID") + .map(id -> FieldId.newBuilder().setName(id).build()) + .collect(Collectors.toList()); + + // Specify when the above fields should be de-identified. + Condition condition = + Condition.newBuilder() + .setField(FieldId.newBuilder().setName("AGE").build()) + .setOperator(RelationalOperator.GREATER_THAN) + .setValue(Value.newBuilder().setIntegerValue(89).build()) + .build(); + // Apply the condition to records + RecordCondition recordCondition = + RecordCondition.newBuilder() + .setExpressions( + Expressions.newBuilder() + .setConditions(Conditions.newBuilder().addConditions(condition).build()) + .build()) + .build(); + + // Associate the de-identification and conditions with the specified fields. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setInfoTypeTransformations(infoTypeTransformations) + .addAllFields(fieldIds) + .setCondition(recordCondition) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println("Table after de-identification: " + response.getItem().getTable()); + + return response.getItem().getTable(); + } + } +} +// [END dlp_deidentify_table_condition_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionMasking.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionMasking.java new file mode 100644 index 00000000000..4da5db4bf41 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableConditionMasking.java @@ -0,0 +1,140 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_condition_masking] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.CharacterMaskConfig; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordCondition; +import com.google.privacy.dlp.v2.RecordCondition.Condition; +import com.google.privacy.dlp.v2.RecordCondition.Conditions; +import com.google.privacy.dlp.v2.RecordCondition.Expressions; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.RelationalOperator; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; + +public class DeIdentifyTableConditionMasking { + + public static void deIdentifyTableConditionMasking() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + + deIdentifyTableConditionMasking(projectId, tableToDeIdentify); + } + + public static Table deIdentifyTableConditionMasking(String projectId, Table tableToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify how the content should be de-identified. + CharacterMaskConfig characterMaskConfig = + CharacterMaskConfig.newBuilder().setMaskingCharacter("*").build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder().setCharacterMaskConfig(characterMaskConfig).build(); + + // Specify field to be de-identified. + FieldId fieldId = FieldId.newBuilder().setName("HAPPINESS SCORE").build(); + + // Specify when the above field should be de-identified. + Condition condition = + Condition.newBuilder() + .setField(FieldId.newBuilder().setName("AGE").build()) + .setOperator(RelationalOperator.GREATER_THAN) + .setValue(Value.newBuilder().setIntegerValue(89).build()) + .build(); + // Apply the condition to records + RecordCondition recordCondition = + RecordCondition.newBuilder() + .setExpressions( + Expressions.newBuilder() + .setConditions(Conditions.newBuilder().addConditions(condition).build()) + .build()) + .build(); + + // Associate the de-identification and conditions with the specified field. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addFields(fieldId) + .setCondition(recordCondition) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println("Table after de-identification: " + response.getItem().getTable()); + + return response.getItem().getTable(); + } + } +} +// [END dlp_deidentify_table_condition_masking] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableInfoTypes.java new file mode 100644 index 00000000000..3ea48cdb2f0 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableInfoTypes.java @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class DeIdentifyTableInfoTypes { + + public static void deIdentifyTableInfoTypes() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "Charles Dickens name was a curse invented by Shakespeare.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in Jane Austen's novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) + .build()) + .build(); + + deIdentifyTableInfoTypes(projectId, tableToDeIdentify); + } + + public static Table deIdentifyTableInfoTypes(String projectId, Table tableToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify how the content should be de-identified. + // Select type of info to be replaced. + InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build(); + // Specify that findings should be replaced with corresponding info type name. + ReplaceWithInfoTypeConfig replaceWithInfoTypeConfig = + ReplaceWithInfoTypeConfig.getDefaultInstance(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(replaceWithInfoTypeConfig) + .build(); + // Associate info type with the replacement strategy + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + InfoTypeTransformations infoTypeTransformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + // Specify fields to be de-identified. + List fieldIds = + Stream.of("PATIENT", "FACTOID") + .map(id -> FieldId.newBuilder().setName(id).build()) + .collect(Collectors.toList()); + + // Associate the de-identification and conditions with the specified field. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setInfoTypeTransformations(infoTypeTransformations) + .addAllFields(fieldIds) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println("Table after de-identification: " + response.getItem().getTable()); + + return response.getItem().getTable(); + } + } +} +// [END dlp_deidentify_table_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableRowSuppress.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableRowSuppress.java new file mode 100644 index 00000000000..0c02dc16649 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableRowSuppress.java @@ -0,0 +1,126 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_row_suppress] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RecordCondition; +import com.google.privacy.dlp.v2.RecordCondition.Condition; +import com.google.privacy.dlp.v2.RecordCondition.Conditions; +import com.google.privacy.dlp.v2.RecordCondition.Expressions; +import com.google.privacy.dlp.v2.RecordSuppression; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.RelationalOperator; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; + +public class DeIdentifyTableRowSuppress { + + public static void deIdentifyTableRowSuppress() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + + deIdentifyTableRowSuppress(projectId, tableToDeIdentify); + } + + public static Table deIdentifyTableRowSuppress(String projectId, Table tableToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify when the content should be de-identified. + Condition condition = + Condition.newBuilder() + .setField(FieldId.newBuilder().setName("AGE").build()) + .setOperator(RelationalOperator.GREATER_THAN) + .setValue(Value.newBuilder().setIntegerValue(89).build()) + .build(); + // Apply the condition to record suppression. + RecordSuppression recordSuppressions = + RecordSuppression.newBuilder() + .setCondition( + RecordCondition.newBuilder() + .setExpressions( + Expressions.newBuilder() + .setConditions( + Conditions.newBuilder().addConditions(condition).build()) + .build()) + .build()) + .build(); + // Use record suppression as the only transformation + RecordTransformations transformations = + RecordTransformations.newBuilder().addRecordSuppressions(recordSuppressions).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println("Table after de-identification: " + response.getItem().getTable()); + + return response.getItem().getTable(); + } + } +} +// [END dlp_deidentify_table_row_suppress] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableWithFpe.java new file mode 100644 index 00000000000..7fb249f6e48 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTableWithFpe.java @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_table_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class DeIdentifyTableWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) + .addHeaders(FieldId.newBuilder().setName("Date").build()) + .addHeaders(FieldId.newBuilder().setName("Compensation").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("11111").build()) + .addValues(Value.newBuilder().setStringValue("2015").build()) + .addValues(Value.newBuilder().setStringValue("$10").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("11111").build()) + .addValues(Value.newBuilder().setStringValue("2016").build()) + .addValues(Value.newBuilder().setStringValue("$20").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22222").build()) + .addValues(Value.newBuilder().setStringValue("2016").build()) + .addValues(Value.newBuilder().setStringValue("$15").build()) + .build()) + .build(); + deIdentifyTableWithFpe(projectId, tableToDeIdentify, kmsKeyName, wrappedAesKey); + } + + public static void deIdentifyTableWithFpe( + String projectId, Table tableToDeIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how the content should be encrypted. + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + + // Specify field to be encrypted. + FieldId fieldId = FieldId.newBuilder().setName("Employee ID").build(); + + // Associate the encryption with the specified field. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addFields(fieldId) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println( + "Table after format-preserving encryption: " + response.getItem().getTable()); + } + } +} +// [END dlp_deidentify_table_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTextWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTextWithFpe.java new file mode 100644 index 00000000000..8922db0b471 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyTextWithFpe.java @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_text_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.Arrays; + +public class DeIdentifyTextWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToDeIdentify = "I'm Gary and my email is gary@example.com"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + deIdentifyTextWithFpe(projectId, textToDeIdentify, kmsKeyName, wrappedAesKey); + } + + public static void deIdentifyTextWithFpe( + String projectId, String textToDeIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to de-identify. + ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build(); + + // Specify the type of info you want the service to de-identify. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + InfoType infoType = InfoType.newBuilder().setName("PHONE_NUMBER").build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(Arrays.asList(infoType)).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it. + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how the info from the inspection should be encrypted. + InfoType surrogateInfoType = InfoType.newBuilder().setName("PHONE_TOKEN").build(); + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .setSurrogateInfoType(surrogateInfoType) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .build(); + InfoTypeTransformations transformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results. + System.out.println( + "Text after format-preserving encryption: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_text_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithDateShift.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithDateShift.java new file mode 100644 index 00000000000..abbc2dae5c3 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithDateShift.java @@ -0,0 +1,167 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_date_shift] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.base.Splitter; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DateShiftConfig; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Value; +import com.google.type.Date; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class DeIdentifyWithDateShift { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Path inputCsvFile = Paths.get("path/to/your/input/file.csv"); + Path outputCsvFile = Paths.get("path/to/your/output/file.csv"); + deIdentifyWithDateShift(projectId, inputCsvFile, outputCsvFile); + } + + public static void deIdentifyWithDateShift( + String projectId, Path inputCsvFile, Path outputCsvFile) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Read the contents of the CSV file into a Table + List headers; + List rows; + try (BufferedReader input = Files.newBufferedReader(inputCsvFile)) { + // Parse and convert the first line into header names + headers = + Arrays.stream(input.readLine().split(",")) + .map(header -> FieldId.newBuilder().setName(header).build()) + .collect(Collectors.toList()); + // Parse the remainder of the file as Table.Rows + rows = + input.lines().map(DeIdentifyWithDateShift::parseLineAsRow).collect(Collectors.toList()); + } + Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build(); + ContentItem item = ContentItem.newBuilder().setTable(table).build(); + + // Set the maximum days to shift dates backwards (lower bound) or forward (upper bound) + DateShiftConfig dateShiftConfig = + DateShiftConfig.newBuilder().setLowerBoundDays(5).setUpperBoundDays(5).build(); + PrimitiveTransformation transformation = + PrimitiveTransformation.newBuilder().setDateShiftConfig(dateShiftConfig).build(); + // Specify which fields the DateShift should apply too + List dateFields = Arrays.asList(headers.get(1), headers.get(3)); + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .addAllFields(dateFields) + .setPrimitiveTransformation(transformation) + .build(); + RecordTransformations recordTransformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + // Specify the config for the de-identify request + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(recordTransformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Write the results to the target CSV file + try (BufferedWriter writer = Files.newBufferedWriter(outputCsvFile)) { + Table outTable = response.getItem().getTable(); + String headerOut = + outTable.getHeadersList().stream() + .map(FieldId::getName) + .collect(Collectors.joining(",")); + writer.write(headerOut + "\n"); + + List rowOutput = + outTable.getRowsList().stream() + .map(row -> joinRow(row.getValuesList())) + .collect(Collectors.toList()); + for (String line : rowOutput) { + writer.write(line + "\n"); + } + System.out.println("Content written to file: " + outputCsvFile.toString()); + } + } + } + + // Convert the string from the csv file into com.google.type.Date + public static Date parseAsDate(String s) { + LocalDate date = LocalDate.parse(s, DateTimeFormatter.ofPattern("MM/dd/yyyy")); + return Date.newBuilder() + .setDay(date.getDayOfMonth()) + .setMonth(date.getMonthValue()) + .setYear(date.getYear()) + .build(); + } + + // Each row is in the format: Name,BirthDate,CreditCardNumber,RegisterDate + public static Table.Row parseLineAsRow(String line) { + List values = Splitter.on(",").splitToList(line); + Value name = Value.newBuilder().setStringValue(values.get(0)).build(); + Value birthDate = Value.newBuilder().setDateValue(parseAsDate(values.get(1))).build(); + Value creditCardNumber = Value.newBuilder().setStringValue(values.get(2)).build(); + Value registerDate = Value.newBuilder().setDateValue(parseAsDate(values.get(3))).build(); + return Table.Row.newBuilder() + .addValues(name) + .addValues(birthDate) + .addValues(creditCardNumber) + .addValues(registerDate) + .build(); + } + + public static String formatDate(Date d) { + return String.format("%s/%s/%s", d.getMonth(), d.getDay(), d.getYear()); + } + + public static String joinRow(List values) { + String name = values.get(0).getStringValue(); + String birthDate = formatDate(values.get(1).getDateValue()); + String creditCardNumber = values.get(2).getStringValue(); + String registerDate = formatDate(values.get(3).getDateValue()); + return String.join(",", name, birthDate, creditCardNumber, registerDate); + } +} +// [END dlp_deidentify_date_shift] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithExceptionList.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithExceptionList.java new file mode 100644 index 00000000000..42c5c25d2d8 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithExceptionList.java @@ -0,0 +1,118 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_exception_list] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import java.io.IOException; + +public class DeIdentifyWithExceptionList { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToDeIdentify = "jack@example.org accessed customer record of user5@example.com"; + deIdentifyWithExceptionList(projectId, textToDeIdentify); + } + + public static void deIdentifyWithExceptionList(String projectId, String textToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + + // Specify what content you want the service to DeIdentify. + ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build(); + + // Construct the custom word list to be detected. + Dictionary wordList = + Dictionary.newBuilder() + .setWordList( + WordList.newBuilder() + .addWords("jack@example.org") + .addWords("jill@example.org") + .build()) + .build(); + + // Construct the custom dictionary detector associated with the word list. + InfoType developerEmail = InfoType.newBuilder().setName("DEVELOPER_EMAIL").build(); + CustomInfoType customInfoType = + CustomInfoType.newBuilder().setInfoType(developerEmail).setDictionary(wordList).build(); + + // Specify the word list custom info type and build-in info type the inspection will look for. + InfoType emailAddress = InfoType.newBuilder().setName("EMAIL_ADDRESS").build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addInfoTypes(emailAddress) + .addCustomInfoTypes(customInfoType) + .build(); + + // Define type of deidentification as replacement. + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(ReplaceWithInfoTypeConfig.getDefaultInstance()) + .build(); + + // Associate de-identification type with info type. + InfoTypeTransformation transformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(emailAddress) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + + // Construct the configuration for the de-id request and list all desired transformations. + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(transformation)) + .build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results + System.out.println( + "Text after replace with infotype config: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_exception_list] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithFpe.java new file mode 100644 index 00000000000..bb947c9a677 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithFpe.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.Arrays; + +public class DeIdentifyWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToDeIdentify = "I'm Gary and my email is gary@example.com"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + deIdentifyWithFpe(projectId, textToDeIdentify, kmsKeyName, wrappedAesKey); + } + + public static void deIdentifyWithFpe( + String projectId, String textToDeIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to DeIdentify + ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("US_SOCIAL_SECURITY_NUMBER").build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(Arrays.asList(infoType)).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how the info from the inspection should be encrypted. + InfoType surrogateInfoType = InfoType.newBuilder().setName("SSN_TOKEN").build(); + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .setSurrogateInfoType(surrogateInfoType) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .build(); + InfoTypeTransformations transformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results + System.out.println( + "Text after format-preserving encryption: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithInfoType.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithInfoType.java new file mode 100644 index 00000000000..0053ab696be --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithInfoType.java @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_infotype] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import java.io.IOException; + +public class DeIdentifyWithInfoType { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "My email is test@example.com"; + deIdentifyWithInfoType(projectId, textToInspect); + } + + public static void deIdentifyWithInfoType(String projectId, String textToRedact) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ContentItem item = ContentItem.newBuilder().setValue(textToRedact).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("EMAIL_ADDRESS").build(); + InspectConfig inspectConfig = InspectConfig.newBuilder().addInfoTypes(infoType).build(); + // Specify replacement string to be used for the finding. + ReplaceWithInfoTypeConfig replaceWithInfoTypeConfig = + ReplaceWithInfoTypeConfig.newBuilder().build(); + // Define type of deidentification as replacement with info type. + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(replaceWithInfoTypeConfig) + .build(); + // Associate deidentification type with info type. + InfoTypeTransformation transformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + // Construct the configuration for the Redact request and list all desired transformations. + DeidentifyConfig redactConfig = + DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(transformation)) + .build(); + + // Construct the Redact request to be sent by the client. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setDeidentifyConfig(redactConfig) + .setInspectConfig(inspectConfig) + .build(); + + // Use the client to send the API request. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Parse the response and process results + System.out.println("Text after redaction: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_infotype] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithMasking.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithMasking.java new file mode 100644 index 00000000000..5d387688c0a --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithMasking.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_masking] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.CharacterMaskConfig; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import java.io.IOException; +import java.util.Arrays; + +public class DeIdentifyWithMasking { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToDeIdentify = "My SSN is 372819127"; + deIdentifyWithMasking(projectId, textToDeIdentify); + } + + public static void deIdentifyWithMasking(String projectId, String textToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + + // Specify what content you want the service to DeIdentify + ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("US_SOCIAL_SECURITY_NUMBER").build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(Arrays.asList(infoType)).build(); + + // Specify how the info from the inspection should be masked. + CharacterMaskConfig characterMaskConfig = + CharacterMaskConfig.newBuilder() + .setMaskingCharacter("X") // Character to replace the found info with + .setNumberToMask(5) // How many characters should be masked + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(ReplaceWithInfoTypeConfig.getDefaultInstance()) + .build(); + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .build(); + InfoTypeTransformations transformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results + System.out.println("Text after masking: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_masking] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithRedaction.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithRedaction.java new file mode 100644 index 00000000000..4c7f748bc1f --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithRedaction.java @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_redact] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RedactConfig; + +public class DeIdentifyWithRedaction { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = + "My name is Alicia Abernathy, and my email address is aabernathy@example.com."; + deIdentifyWithRedaction(projectId, textToInspect); + } + + // Inspects the provided text. + public static void deIdentifyWithRedaction(String projectId, String textToRedact) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ContentItem item = ContentItem.newBuilder().setValue(textToRedact).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("EMAIL_ADDRESS").build(); + InspectConfig inspectConfig = InspectConfig.newBuilder().addInfoTypes(infoType).build(); + // Define type of deidentification. + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setRedactConfig(RedactConfig.getDefaultInstance()) + .build(); + // Associate deidentification type with info type. + InfoTypeTransformation transformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + // Construct the configuration for the Redact request and list all desired transformations. + DeidentifyConfig redactConfig = + DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(transformation)) + .build(); + + // Construct the Redact request to be sent by the client. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setDeidentifyConfig(redactConfig) + .setInspectConfig(inspectConfig) + .build(); + + // Use the client to send the API request. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Parse the response and process results + System.out.println("Text after redaction: " + response.getItem().getValue()); + } catch (Exception e) { + System.out.println("Error during inspectString: \n" + e.toString()); + } + } +} +// [END dlp_deidentify_redact] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithReplacement.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithReplacement.java new file mode 100644 index 00000000000..3a578f05851 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithReplacement.java @@ -0,0 +1,98 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_replace] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReplaceValueConfig; +import com.google.privacy.dlp.v2.Value; + +public class DeIdentifyWithReplacement { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = + "My name is Alicia Abernathy, and my email address is aabernathy@example.com."; + deIdentifyWithReplacement(projectId, textToInspect); + } + + // Inspects the provided text. + public static void deIdentifyWithReplacement(String projectId, String textToRedact) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ContentItem item = ContentItem.newBuilder().setValue(textToRedact).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("EMAIL_ADDRESS").build(); + InspectConfig inspectConfig = InspectConfig.newBuilder().addInfoTypes(infoType).build(); + // Specify replacement string to be used for the finding. + ReplaceValueConfig replaceValueConfig = + ReplaceValueConfig.newBuilder() + .setNewValue(Value.newBuilder().setStringValue("[email-address]").build()) + .build(); + // Define type of deidentification as replacement. + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder().setReplaceConfig(replaceValueConfig).build(); + // Associate deidentification type with info type. + InfoTypeTransformation transformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + // Construct the configuration for the Redact request and list all desired transformations. + DeidentifyConfig redactConfig = + DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(transformation)) + .build(); + + // Construct the Redact request to be sent by the client. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setDeidentifyConfig(redactConfig) + .setInspectConfig(inspectConfig) + .build(); + + // Use the client to send the API request. + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Parse the response and process results + System.out.println("Text after redaction: " + response.getItem().getValue()); + } catch (Exception e) { + System.out.println("Error during inspectString: \n" + e.toString()); + } + } +} +// [END dlp_deidentify_replace] diff --git a/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithSimpleWordList.java b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithSimpleWordList.java new file mode 100644 index 00000000000..993277a0b90 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/DeIdentifyWithSimpleWordList.java @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_deidentify_simple_word_list] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig; +import java.io.IOException; + +public class DeIdentifyWithSimpleWordList { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToDeIdentify = "Patient was seen in RM-YELLOW then transferred to rm green."; + deidentifyWithSimpleWordList(projectId, textToDeIdentify); + } + + public static void deidentifyWithSimpleWordList(String projectId, String textToDeIdentify) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + + // Specify what content you want the service to DeIdentify. + ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build(); + + // Construct the word list to be detected + Dictionary wordList = + Dictionary.newBuilder() + .setWordList( + WordList.newBuilder() + .addWords("RM-GREEN") + .addWords("RM-YELLOW") + .addWords("RM-ORANGE") + .build()) + .build(); + + // Specify the word list custom info type the inspection will look for. + InfoType infoType = InfoType.newBuilder().setName("CUSTOM_ROOM_ID").build(); + CustomInfoType customInfoType = + CustomInfoType.newBuilder().setInfoType(infoType).setDictionary(wordList).build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addCustomInfoTypes(customInfoType).build(); + + // Define type of deidentification as replacement. + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setReplaceWithInfoTypeConfig(ReplaceWithInfoTypeConfig.getDefaultInstance()) + .build(); + + // Associate deidentification type with info type. + InfoTypeTransformation transformation = + InfoTypeTransformation.newBuilder() + .addInfoTypes(infoType) + .setPrimitiveTransformation(primitiveTransformation) + .build(); + + // Construct the configuration for the Redact request and list all desired transformations. + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder() + .setInfoTypeTransformations( + InfoTypeTransformations.newBuilder().addTransformations(transformation)) + .build(); + + // Combine configurations into a request for the service. + DeidentifyContentRequest request = + DeidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setDeidentifyConfig(deidentifyConfig) + .build(); + + // Send the request and receive response from the service + DeidentifyContentResponse response = dlp.deidentifyContent(request); + + // Print the results + System.out.println( + "Text after replace with infotype config: " + response.getItem().getValue()); + } + } +} +// [END dlp_deidentify_simple_word_list] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InfoTypesList.java b/dlp/snippets/src/main/java/dlp/snippets/InfoTypesList.java new file mode 100644 index 00000000000..43a304ead9a --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InfoTypesList.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_list_info_types] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.InfoTypeDescription; +import com.google.privacy.dlp.v2.ListInfoTypesRequest; +import com.google.privacy.dlp.v2.ListInfoTypesResponse; +import java.io.IOException; + +public class InfoTypesList { + + public static void main(String[] args) throws IOException { + listInfoTypes(); + } + + // Lists the types of sensitive information the DLP API supports. + public static void listInfoTypes() throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpClient = DlpServiceClient.create()) { + + // Construct the request to be sent by the client + ListInfoTypesRequest listInfoTypesRequest = + ListInfoTypesRequest.newBuilder() + // Only return infoTypes supported by certain parts of the API. + // Supported filters are "supported_by=INSPECT" and "supported_by=RISK_ANALYSIS" + // Defaults to "supported_by=INSPECT" + .setFilter("supported_by=INSPECT") + // BCP-47 language code for localized infoType friendly names. + // Defaults to "en_US" + .setLanguageCode("en-US") + .build(); + + // Use the client to send the API request. + ListInfoTypesResponse response = dlpClient.listInfoTypes(listInfoTypesRequest); + + // Parse the response and process the results + System.out.println("Infotypes found:"); + for (InfoTypeDescription infoTypeDescription : response.getInfoTypesList()) { + System.out.println("Name : " + infoTypeDescription.getName()); + System.out.println("Display name : " + infoTypeDescription.getDisplayName()); + } + } + } +} +// [END dlp_list_info_types] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTable.java b/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTable.java new file mode 100644 index 00000000000..3d7c31203fa --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTable.java @@ -0,0 +1,179 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_bigquery] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.BigQueryOptions; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeStats; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectDataSourceDetails; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class InspectBigQueryTable { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String bigQueryDatasetId = "your-bigquery-dataset-id"; + String bigQueryTableId = "your-bigquery-table-id"; + String topicId = "your-pubsub-topic-id"; + String subscriptionId = "your-pubsub-subscription-id"; + inspectBigQueryTable(projectId, bigQueryDatasetId, bigQueryTableId, topicId, subscriptionId); + } + + // Inspects a BigQuery Table + public static void inspectBigQueryTable( + String projectId, + String bigQueryDatasetId, + String bigQueryTableId, + String topicId, + String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the BigQuery table to be inspected. + BigQueryTable tableReference = + BigQueryTable.newBuilder() + .setProjectId(projectId) + .setDatasetId(bigQueryDatasetId) + .setTableId(bigQueryTableId) + .build(); + + BigQueryOptions bigQueryOptions = + BigQueryOptions.newBuilder().setTableReference(tableReference).build(); + + StorageConfig storageConfig = + StorageConfig.newBuilder().setBigQueryOptions(bigQueryOptions).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + // Specify how the content should be inspected. + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Specify the action that is triggered when the job completes. + String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId); + Action.PublishToPubSub publishToPubSub = + Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the long running job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(inspectConfig) + .addActions(action) + .build(); + + // Create the request for the job configured above. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Use the client to send the request. + final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest); + System.out.println("Job created: " + dlpJob.getName()); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Get the latest state of the job from the service + GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + DlpJob completedJob = dlp.getDlpJob(request); + + // Parse the response and process results. + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult(); + System.out.println("Findings: "); + for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) { + System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName()); + System.out.println("\tCount: " + infoTypeStat.getCount()); + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_inspect_bigquery] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTableWithSampling.java b/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTableWithSampling.java new file mode 100644 index 00000000000..30e3f5ec663 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectBigQueryTableWithSampling.java @@ -0,0 +1,174 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_bigquery_with_sampling] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.BigQueryOptions; +import com.google.privacy.dlp.v2.BigQueryOptions.SampleMethod; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeStats; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectDataSourceDetails; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class InspectBigQueryTableWithSampling { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String topicId = "your-pubsub-topic-id"; + String subscriptionId = "your-pubsub-subscription-id"; + inspectBigQueryTableWithSampling(projectId, topicId, subscriptionId); + } + + // Inspects a BigQuery Table + public static void inspectBigQueryTableWithSampling( + String projectId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the BigQuery table to be inspected. + BigQueryTable tableReference = + BigQueryTable.newBuilder() + .setProjectId("bigquery-public-data") + .setDatasetId("usa_names") + .setTableId("usa_1910_current") + .build(); + + BigQueryOptions bigQueryOptions = + BigQueryOptions.newBuilder() + .setTableReference(tableReference) + .setRowsLimit(1000) + .setSampleMethod(SampleMethod.RANDOM_START) + .addIdentifyingFields(FieldId.newBuilder().setName("name")) + .build(); + + StorageConfig storageConfig = + StorageConfig.newBuilder().setBigQueryOptions(bigQueryOptions).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build(); + + // Specify how the content should be inspected. + InspectConfig inspectConfig = + InspectConfig.newBuilder().addInfoTypes(infoType).setIncludeQuote(true).build(); + + // Specify the action that is triggered when the job completes. + String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId); + Action.PublishToPubSub publishToPubSub = + Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the long running job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(inspectConfig) + .addActions(action) + .build(); + + // Create the request for the job configured above. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Use the client to send the request. + final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest); + System.out.println("Job created: " + dlpJob.getName()); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Get the latest state of the job from the service + GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + DlpJob completedJob = dlp.getDlpJob(request); + + // Parse the response and process results. + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult(); + System.out.println("Findings: "); + for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) { + System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName()); + System.out.println("\tCount: " + infoTypeStat.getCount()); + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_inspect_bigquery_with_sampling] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectDatastoreEntity.java b/dlp/snippets/src/main/java/dlp/snippets/InspectDatastoreEntity.java new file mode 100644 index 00000000000..c70f22bd3a4 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectDatastoreEntity.java @@ -0,0 +1,180 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_datastore] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DatastoreOptions; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeStats; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectDataSourceDetails; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.KindExpression; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PartitionId; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class InspectDatastoreEntity { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datastoreNamespace = "your-datastore-namespace"; + String datastoreKind = "your-datastore-kind"; + String topicId = "your-pubsub-topic-id"; + String subscriptionId = "your-pubsub-subscription-id"; + insepctDatastoreEntity(projectId, datastoreNamespace, datastoreKind, topicId, subscriptionId); + } + + // Inspects a Datastore Entity. + public static void insepctDatastoreEntity( + String projectId, + String datastoreNamespce, + String datastoreKind, + String topicId, + String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the Datastore entity to be inspected. + PartitionId partitionId = + PartitionId.newBuilder() + .setProjectId(projectId) + .setNamespaceId(datastoreNamespce) + .build(); + KindExpression kindExpression = KindExpression.newBuilder().setName(datastoreKind).build(); + + DatastoreOptions datastoreOptions = + DatastoreOptions.newBuilder().setKind(kindExpression).setPartitionId(partitionId).build(); + + StorageConfig storageConfig = + StorageConfig.newBuilder().setDatastoreOptions(datastoreOptions).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + // Specify how the content should be inspected. + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Specify the action that is triggered when the job completes. + String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId); + Action.PublishToPubSub publishToPubSub = + Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the long running job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(inspectConfig) + .addActions(action) + .build(); + + // Create the request for the job configured above. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Use the client to send the request. + final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest); + System.out.println("Job created: " + dlpJob.getName()); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Get the latest state of the job from the service + GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + DlpJob completedJob = dlp.getDlpJob(request); + + // Parse the response and process results. + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult(); + System.out.println("Findings: "); + for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) { + System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName()); + System.out.println("\tCount: " + infoTypeStat.getCount()); + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_inspect_datastore] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFile.java b/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFile.java new file mode 100644 index 00000000000..758464dc17b --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFile.java @@ -0,0 +1,167 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_gcs] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CloudStorageOptions.FileSet; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeStats; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectDataSourceDetails; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class InspectGcsFile { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String gcsUri = "gs://" + "your-bucket-name" + "/path/to/your/file.txt"; + String topicId = "your-pubsub-topic-id"; + String subscriptionId = "your-pubsub-subscription-id"; + inspectGcsFile(projectId, gcsUri, topicId, subscriptionId); + } + + // Inspects a file in a Google Cloud Storage Bucket. + public static void inspectGcsFile( + String projectId, String gcsUri, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the GCS file to be inspected. + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder().setFileSet(FileSet.newBuilder().setUrl(gcsUri)).build(); + + StorageConfig storageConfig = + StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + // Specify how the content should be inspected. + InspectConfig inspectConfig = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Specify the action that is triggered when the job completes. + String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId); + Action.PublishToPubSub publishToPubSub = + Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the long running job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(inspectConfig) + .addActions(action) + .build(); + + // Create the request for the job configured above. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Use the client to send the request. + final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest); + System.out.println("Job created: " + dlpJob.getName()); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Get the latest state of the job from the service + GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + DlpJob completedJob = dlp.getDlpJob(request); + + // Parse the response and process results. + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult(); + System.out.println("Findings: "); + for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) { + System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName()); + System.out.println("\tCount: " + infoTypeStat.getCount()); + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_inspect_gcs] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFileWithSampling.java b/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFileWithSampling.java new file mode 100644 index 00000000000..1c4078587d4 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectGcsFileWithSampling.java @@ -0,0 +1,175 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_gcs_with_sampling] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CloudStorageOptions.FileSet; +import com.google.privacy.dlp.v2.CloudStorageOptions.SampleMethod; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FileType; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeStats; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectDataSourceDetails; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class InspectGcsFileWithSampling { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String gcsUri = "gs://" + "your-bucket-name" + "/path/to/your/file.txt"; + String topicId = "your-pubsub-topic-id"; + String subscriptionId = "your-pubsub-subscription-id"; + inspectGcsFileWithSampling(projectId, gcsUri, topicId, subscriptionId); + } + + // Inspects a file in a Google Cloud Storage Bucket. + public static void inspectGcsFileWithSampling( + String projectId, String gcsUri, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the GCS file to be inspected and sampling configuration + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder() + .setFileSet(FileSet.newBuilder().setUrl(gcsUri)) + .setBytesLimitPerFile(200) + .addFileTypes(FileType.TEXT_FILE) + .setFilesLimitPercent(90) + .setSampleMethod(SampleMethod.RANDOM_START) + .build(); + + StorageConfig storageConfig = + StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("PERSON_NAME").build(); + + // Specify how the content should be inspected. + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addInfoTypes(infoType) + .setExcludeInfoTypes(true) + .setIncludeQuote(true) + .setMinLikelihood(Likelihood.POSSIBLE) + .build(); + + // Specify the action that is triggered when the job completes. + String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId); + Action.PublishToPubSub publishToPubSub = + Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the long running job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(inspectConfig) + .addActions(action) + .build(); + + // Create the request for the job configured above. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Use the client to send the request. + final DlpJob dlpJob = dlp.createDlpJob(createDlpJobRequest); + System.out.println("Job created: " + dlpJob.getName()); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Get the latest state of the job from the service + GetDlpJobRequest request = GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + DlpJob completedJob = dlp.getDlpJob(request); + + // Parse the response and process results. + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + InspectDataSourceDetails.Result result = completedJob.getInspectDetails().getResult(); + System.out.println("Findings: "); + for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) { + System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName()); + System.out.println("\tCount: " + infoTypeStat.getCount()); + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_inspect_gcs_with_sampling] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectImageFile.java b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFile.java new file mode 100644 index 00000000000..2251abbff46 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFile.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_image_file] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectImageFile { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String filePath = "path/to/image.png"; + inspectImageFile(projectId, filePath); + } + + // Inspects the specified image file. + public static void inspectImageFile(String projectId, String filePath) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(filePath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE).setData(fileBytes).build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results. + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_image_file] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileAllInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileAllInfoTypes.java new file mode 100644 index 00000000000..87847fb7b6e --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileAllInfoTypes.java @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_image_all_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.IOException; + +class InspectImageFileAllInfoTypes { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/sensitive-data-image.jpeg"; + inspectImageFileAllInfoTypes(projectId, inputPath); + } + + static void inspectImageFileAllInfoTypes(String projectId, String inputPath) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Construct the Inspect request to be sent by the client. + // Do not specify the type of info to inspect. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(ContentItem.newBuilder().setByteItem(byteItem).build()) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results. + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_image_all_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileListedInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileListedInfoTypes.java new file mode 100644 index 00000000000..85aa82ac186 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectImageFileListedInfoTypes.java @@ -0,0 +1,89 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_image_listed_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +class InspectImageFileListedInfoTypes { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/sensitive-data-image.jpeg"; + inspectImageFileListedInfoTypes(projectId, inputPath); + } + + static void inspectImageFileListedInfoTypes(String projectId, String inputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Specify the type of info the inspection will look for. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : + new String[] {"US_SOCIAL_SECURITY_NUMBER", "EMAIL_ADDRESS", "PHONE_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Construct the configuration for the Inspect request. + InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(ContentItem.newBuilder().setByteItem(byteItem).build()) + .setInspectConfig(inspectConfig) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results. + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_image_listed_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectPhoneNumber.java b/dlp/snippets/src/main/java/dlp/snippets/InspectPhoneNumber.java new file mode 100644 index 00000000000..c707088cc3c --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectPhoneNumber.java @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_phone_number] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; + +public class InspectPhoneNumber { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "My name is Gary and my email is gary@example.com"; + inspectString(projectId, textToInspect); + } + + // Inspects the provided text. + public static void inspectString(String projectId, String textToInspect) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ContentItem item = ContentItem.newBuilder().setValue(textToInspect).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("PHONE_NUMBER").build(); + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder() + .setIncludeQuote(true) + .setMinLikelihood(Likelihood.POSSIBLE) + .addInfoTypes(infoType) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_phone_number] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectString.java b/dlp/snippets/src/main/java/dlp/snippets/InspectString.java new file mode 100644 index 00000000000..28f0d08628f --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectString.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectString { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "My name is Gary and my email is gary@example.com"; + inspectString(projectId, textToInspect); + } + + // Inspects the provided text. + public static void inspectString(String projectId, String textToInspect) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomExcludingSubstring.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomExcludingSubstring.java new file mode 100644 index 00000000000..015367d9708 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomExcludingSubstring.java @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_custom_excluding_substring] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class InspectStringCustomExcludingSubstring { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Name: Doe, John. Name: Example, Jimmy"; + String customDetectorPattern = "[A-Z][a-z]{1,15}, [A-Z][a-z]{1,15}"; + List excludedSubstringList = Arrays.asList("Jimmy"); + inspectStringCustomExcludingSubstring( + projectId, textToInspect, customDetectorPattern, excludedSubstringList); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringCustomExcludingSubstring( + String projectId, + String textToInspect, + String customDetectorPattern, + List excludedSubstringList) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + InfoType infoType = InfoType.newBuilder().setName("CUSTOM_NAME_DETECTOR").build(); + CustomInfoType customInfoType = + CustomInfoType.newBuilder() + .setInfoType(infoType) + .setRegex(Regex.newBuilder().setPattern(customDetectorPattern)) + .build(); + + // Exclude partial matches from the specified excludedSubstringList. + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .setDictionary( + Dictionary.newBuilder() + .setWordList(WordList.newBuilder().addAllWords(excludedSubstringList))) + .build(); + + // Construct a ruleset that applies the exclusion rule to the EMAIL_ADDRESSES infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(infoType) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addCustomInfoTypes(customInfoType) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_custom_excluding_substring] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomHotword.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomHotword.java new file mode 100644 index 00000000000..254d8f86413 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomHotword.java @@ -0,0 +1,112 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_custom_hotword] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.HotwordRule; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.LikelihoodAdjustment; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.Proximity; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class InspectStringCustomHotword { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "patient name: John Doe"; + String customHotword = "patient"; + inspectStringCustomHotword(projectId, textToInspect, customHotword); + } + + // Inspects the provided text. + public static void inspectStringCustomHotword( + String projectId, String textToInspect, String customHotword) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Increase likelihood of matches that have customHotword nearby + HotwordRule hotwordRule = + HotwordRule.newBuilder() + .setHotwordRegex(Regex.newBuilder().setPattern(customHotword)) + .setProximity(Proximity.newBuilder().setWindowBefore(50)) + .setLikelihoodAdjustment( + LikelihoodAdjustment.newBuilder().setFixedLikelihood(Likelihood.VERY_LIKELY)) + .build(); + + // Construct a ruleset that applies the hotword rule to the PERSON_NAME infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME").build()) + .addRules(InspectionRule.newBuilder().setHotwordRule(hotwordRule)) + .build(); + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME").build()) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .setMinLikelihood(Likelihood.VERY_LIKELY) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_custom_hotword] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomOmitOverlap.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomOmitOverlap.java new file mode 100644 index 00000000000..0cf017f3436 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringCustomOmitOverlap.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_custom_omit_overlap] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.ExclusionType; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.ExcludeInfoTypes; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class InspectStringCustomOmitOverlap { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Name: Jane Doe. Name: Larry Page."; + inspectStringCustomOmitOverlap(projectId, textToInspect); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringCustomOmitOverlap(String projectId, String textToInspect) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Construct the custom infotype. + CustomInfoType customInfoType = + CustomInfoType.newBuilder() + .setInfoType(InfoType.newBuilder().setName("VIP_DETECTOR")) + .setRegex(Regex.newBuilder().setPattern("Larry Page|Sergey Brin")) + .setExclusionType(ExclusionType.EXCLUSION_TYPE_EXCLUDE) + .build(); + + // Exclude matches that also match the custom infotype. + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setExcludeInfoTypes( + ExcludeInfoTypes.newBuilder().addInfoTypes(customInfoType.getInfoType())) + .setMatchingType(MatchingType.MATCHING_TYPE_FULL_MATCH) + .build(); + + // Construct a ruleset that applies the exclusion rule to the PERSON_NAME infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME")) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME")) + .addCustomInfoTypes(customInfoType) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_custom_omit_overlap] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringMultipleRules.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringMultipleRules.java new file mode 100644 index 00000000000..821cfab1117 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringMultipleRules.java @@ -0,0 +1,139 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_multiple_rules] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.HotwordRule; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.LikelihoodAdjustment; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.Proximity; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class InspectStringMultipleRules { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "patient: Jane Doe"; + inspectStringMultipleRules(projectId, textToInspect); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringMultipleRules(String projectId, String textToInspect) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Construct hotword rules + HotwordRule patientRule = + HotwordRule.newBuilder() + .setHotwordRegex(Regex.newBuilder().setPattern("patient")) + .setProximity(Proximity.newBuilder().setWindowBefore(10)) + .setLikelihoodAdjustment( + LikelihoodAdjustment.newBuilder().setFixedLikelihood(Likelihood.VERY_LIKELY)) + .build(); + + HotwordRule doctorRule = + HotwordRule.newBuilder() + .setHotwordRegex(Regex.newBuilder().setPattern("doctor")) + .setProximity(Proximity.newBuilder().setWindowBefore(10)) + .setLikelihoodAdjustment( + LikelihoodAdjustment.newBuilder().setFixedLikelihood(Likelihood.UNLIKELY)) + .build(); + + // Construct exclusion rules + ExclusionRule quasimodoRule = + ExclusionRule.newBuilder() + .setDictionary( + Dictionary.newBuilder().setWordList(WordList.newBuilder().addWords("Quasimodo"))) + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .build(); + + ExclusionRule redactedRule = + ExclusionRule.newBuilder() + .setRegex(Regex.newBuilder().setPattern("REDACTED")) + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .build(); + + // Construct a ruleset that applies the rules to the PERSON_NAME infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME")) + .addRules(InspectionRule.newBuilder().setHotwordRule(patientRule)) + .addRules(InspectionRule.newBuilder().setHotwordRule(doctorRule)) + .addRules(InspectionRule.newBuilder().setExclusionRule(quasimodoRule)) + .addRules(InspectionRule.newBuilder().setExclusionRule(redactedRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME")) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_multiple_rules] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringOmitOverlap.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringOmitOverlap.java new file mode 100644 index 00000000000..83e83076802 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringOmitOverlap.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_omit_overlap] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.ExcludeInfoTypes; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectStringOmitOverlap { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "james@example.com"; + inspectStringOmitOverlap(projectId, textToInspect); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringOmitOverlap(String projectId, String textToInspect) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + List infoTypes = new ArrayList<>(); + for (String typeName : new String[] {"PERSON_NAME", "EMAIL_ADDRESS"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Exclude EMAIL_ADDRESS matches + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setExcludeInfoTypes( + ExcludeInfoTypes.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("EMAIL_ADDRESS"))) + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .build(); + + // Construct a ruleset that applies the exclusion rule to the PERSON_NAME infotype. + // If a PERSON_NAME match overlaps with an EMAIL_ADDRESS match, the PERSON_NAME match will + // be excluded. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("PERSON_NAME")) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_omit_overlap] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDict.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDict.java new file mode 100644 index 00000000000..3db325814d2 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDict.java @@ -0,0 +1,120 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_with_exclusion_dict] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InspectStringWithExclusionDict { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Some email addresses: gary@example.com, example@example.com"; + List excludedMatchList = Arrays.asList("example@example.com"); + inspectStringWithExclusionDict(projectId, textToInspect, excludedMatchList); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringWithExclusionDict( + String projectId, String textToInspect, List excludedMatchList) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + List infoTypes = new ArrayList<>(); + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Exclude matches from the specified excludedMatchList. + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setMatchingType(MatchingType.MATCHING_TYPE_FULL_MATCH) + .setDictionary( + Dictionary.newBuilder() + .setWordList(WordList.newBuilder().addAllWords(excludedMatchList))) + .build(); + + // Construct a ruleset that applies the exclusion rule to the EMAIL_ADDRESSES infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("EMAIL_ADDRESS")) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_with_exclusion_dict] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDictSubstring.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDictSubstring.java new file mode 100644 index 00000000000..0fc065f07be --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionDictSubstring.java @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_with_exclusion_dict_substring] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary; +import com.google.privacy.dlp.v2.CustomInfoType.Dictionary.WordList; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InspectStringWithExclusionDictSubstring { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Some email addresses: gary@example.com, TEST@example.com"; + List excludedSubstringList = Arrays.asList("TEST"); + inspectStringWithExclusionDictSubstring(projectId, textToInspect, excludedSubstringList); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringWithExclusionDictSubstring( + String projectId, String textToInspect, List excludedSubstringList) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + List infoTypes = new ArrayList<>(); + for (String typeName : + new String[] {"EMAIL_ADDRESS", "DOMAIN_NAME", "PHONE_NUMBER", "PERSON_NAME"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Exclude partial matches from the specified excludedSubstringList. + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .setDictionary( + Dictionary.newBuilder() + .setWordList(WordList.newBuilder().addAllWords(excludedSubstringList))) + .build(); + + // Construct a ruleset that applies the exclusion rule to the EMAIL_ADDRESSES infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addAllInfoTypes(infoTypes) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_with_exclusion_dict_substring] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionRegex.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionRegex.java new file mode 100644 index 00000000000..f609a752986 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithExclusionRegex.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_with_exclusion_regex] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectStringWithExclusionRegex { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Some email addresses: gary@example.com, bob@example.org"; + String excludedRegex = ".+@example.com"; + inspectStringWithExclusionRegex(projectId, textToInspect, excludedRegex); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringWithExclusionRegex( + String projectId, String textToInspect, String excludedRegex) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + List infoTypes = new ArrayList<>(); + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Exclude matches from the specified excludedMatchList. + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setMatchingType(MatchingType.MATCHING_TYPE_FULL_MATCH) + .setRegex(Regex.newBuilder().setPattern(excludedRegex)) + .build(); + + // Construct a ruleset that applies the exclusion rule to the EMAIL_ADDRESSES infotype. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("EMAIL_ADDRESS")) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_with_exclusion_regex] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithoutOverlap.java b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithoutOverlap.java new file mode 100644 index 00000000000..a1fc60e2226 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectStringWithoutOverlap.java @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_string_without_overlap] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.ExclusionType; +import com.google.privacy.dlp.v2.ExcludeInfoTypes; +import com.google.privacy.dlp.v2.ExclusionRule; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.MatchingType; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectStringWithoutOverlap { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "example.com is a domain, james@example.org is an email."; + inspectStringWithoutOverlap(projectId, textToInspect); + } + + // Inspects the provided text, avoiding matches specified in the exclusion list. + public static void inspectStringWithoutOverlap(String projectId, String textToInspect) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types. + List infoTypes = new ArrayList<>(); + for (String typeName : new String[] {"DOMAIN_NAME", "EMAIL_ADDRESS"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Define a custom info type to exclude email addresses + CustomInfoType customInfoType = + CustomInfoType.newBuilder() + .setInfoType(InfoType.newBuilder().setName("EMAIL_ADDRESS")) + .setExclusionType(ExclusionType.EXCLUSION_TYPE_EXCLUDE) + .build(); + + // Exclude EMAIL_ADDRESS matches + ExclusionRule exclusionRule = + ExclusionRule.newBuilder() + .setExcludeInfoTypes( + ExcludeInfoTypes.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("EMAIL_ADDRESS"))) + .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH) + .build(); + + // Construct a ruleset that applies the exclusion rule to the DOMAIN_NAME infotype. + // If a DOMAIN_NAME match is part of an EMAIL_ADDRESS match, the DOMAIN_NAME match will + // be excluded. + InspectionRuleSet ruleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(InfoType.newBuilder().setName("DOMAIN_NAME")) + .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule)) + .build(); + + // Construct the configuration for the Inspect request, including the ruleset. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .addCustomInfoTypes(customInfoType) + .setIncludeQuote(true) + .addRuleSet(ruleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_string_without_overlap] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectTable.java b/dlp/snippets/src/main/java/dlp/snippets/InspectTable.java new file mode 100644 index 00000000000..ad015500fc4 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectTable.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_table] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; + +public class InspectTable { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + Table tableToInspect = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("name").build()) + .addHeaders(FieldId.newBuilder().setName("phone").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("John Doe").build()) + .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build())) + .build(); + + inspectTable(projectId, tableToInspect); + } + + // Inspects the provided text. + public static void inspectTable(String projectId, Table tableToInspect) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the table to be inspected. + ContentItem item = ContentItem.newBuilder().setTable(tableToInspect).build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + InfoType infoType = InfoType.newBuilder().setName("PHONE_NUMBER").build(); + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder().addInfoTypes(infoType).setIncludeQuote(true).build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } catch (Exception e) { + System.out.println("Error during inspectString: \n" + e.toString()); + } + } +} +// [END dlp_inspect_table] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectTextFile.java b/dlp/snippets/src/main/java/dlp/snippets/InspectTextFile.java new file mode 100644 index 00000000000..872ecd94356 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectTextFile.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_file] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class InspectTextFile { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String filePath = "path/to/file.txt"; + inspectTextFile(projectId, filePath); + } + + // Inspects the specified text file. + public static void inspectTextFile(String projectId, String filePath) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(filePath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.TEXT_UTF8).setData(fileBytes).build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the type of info the inspection will look for. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setIncludeQuote(true).build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_file] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectWithCustomRegex.java b/dlp/snippets/src/main/java/dlp/snippets/InspectWithCustomRegex.java new file mode 100644 index 00000000000..16aba55c17b --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectWithCustomRegex.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_custom_regex] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class InspectWithCustomRegex { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Patients MRN 444-5-22222"; + String customRegexPattern = "[1-9]{3}-[1-9]{1}-[1-9]{5}"; + inspectWithCustomRegex(projectId, textToInspect, customRegexPattern); + } + + // Inspects a BigQuery Table + public static void inspectWithCustomRegex( + String projectId, String textToInspect, String customRegexPattern) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the regex pattern the inspection will look for. + Regex regex = Regex.newBuilder().setPattern(customRegexPattern).build(); + + // Construct the custom regex detector. + InfoType infoType = InfoType.newBuilder().setName("C_MRN").build(); + CustomInfoType customInfoType = + CustomInfoType.newBuilder().setInfoType(infoType).setRegex(regex).build(); + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder() + .addCustomInfoTypes(customInfoType) + .setIncludeQuote(true) + .setMinLikelihood(Likelihood.POSSIBLE) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_custom_regex] diff --git a/dlp/snippets/src/main/java/dlp/snippets/InspectWithHotwordRules.java b/dlp/snippets/src/main/java/dlp/snippets/InspectWithHotwordRules.java new file mode 100644 index 00000000000..39c253279bf --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/InspectWithHotwordRules.java @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_inspect_hotword_rule] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.HotwordRule; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.LikelihoodAdjustment; +import com.google.privacy.dlp.v2.CustomInfoType.DetectionRule.Proximity; +import com.google.privacy.dlp.v2.CustomInfoType.Regex; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectionRule; +import com.google.privacy.dlp.v2.InspectionRuleSet; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class InspectWithHotwordRules { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToInspect = "Patient's MRN 444-5-22222 and just a number 333-2-33333"; + String customRegexPattern = "[1-9]{3}-[1-9]{1}-[1-9]{5}"; + String hotwordRegexPattern = "(?i)(mrn|medical)(?-i)"; + inspectWithHotwordRules(projectId, textToInspect, customRegexPattern, hotwordRegexPattern); + } + + // Inspects a BigQuery Table + public static void inspectWithHotwordRules( + String projectId, String textToInspect, String customRegexPattern, String hotwordRegexPattern) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the type and content to be inspected. + ByteContentItem byteItem = + ByteContentItem.newBuilder() + .setType(BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(textToInspect)) + .build(); + ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build(); + + // Specify the regex pattern the inspection will look for. + Regex regex = Regex.newBuilder().setPattern(customRegexPattern).build(); + + // Construct the custom regex detector. + InfoType infoType = InfoType.newBuilder().setName("C_MRN").build(); + CustomInfoType customInfoType = + CustomInfoType.newBuilder().setInfoType(infoType).setRegex(regex).build(); + + // Specify hotword likelihood adjustment. + LikelihoodAdjustment likelihoodAdjustment = + LikelihoodAdjustment.newBuilder().setFixedLikelihood(Likelihood.VERY_LIKELY).build(); + + // Specify a window around a finding to apply a detection rule. + Proximity proximity = Proximity.newBuilder().setWindowBefore(10).build(); + + // Construct hotword rule. + HotwordRule hotwordRule = + HotwordRule.newBuilder() + .setHotwordRegex(Regex.newBuilder().setPattern(hotwordRegexPattern).build()) + .setLikelihoodAdjustment(likelihoodAdjustment) + .setProximity(proximity) + .build(); + + // Construct rule set for the inspect config. + InspectionRuleSet inspectionRuleSet = + InspectionRuleSet.newBuilder() + .addInfoTypes(infoType) + .addRules(InspectionRule.newBuilder().setHotwordRule(hotwordRule)) + .build(); + + // Construct the configuration for the Inspect request. + InspectConfig config = + InspectConfig.newBuilder() + .addCustomInfoTypes(customInfoType) + .setIncludeQuote(true) + .setMinLikelihood(Likelihood.POSSIBLE) + .addRuleSet(inspectionRuleSet) + .build(); + + // Construct the Inspect request to be sent by the client. + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(item) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + InspectContentResponse response = dlp.inspectContent(request); + + // Parse the response and process results + System.out.println("Findings: " + response.getResult().getFindingsCount()); + for (Finding f : response.getResult().getFindingsList()) { + System.out.println("\tQuote: " + f.getQuote()); + System.out.println("\tInfo type: " + f.getInfoType().getName()); + System.out.println("\tLikelihood: " + f.getLikelihood()); + } + } + } +} +// [END dlp_inspect_hotword_rule] diff --git a/dlp/snippets/src/main/java/dlp/snippets/JobsCreate.java b/dlp/snippets/src/main/java/dlp/snippets/JobsCreate.java new file mode 100644 index 00000000000..54325579721 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/JobsCreate.java @@ -0,0 +1,120 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_create_job] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.privacy.dlp.v2.StorageConfig.TimespanConfig; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class JobsCreate { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String gcsPath = "gs://" + "your-bucket-name" + "path/to/file.txt"; + createJobs(projectId, gcsPath); + } + + // Creates a DLP Job + public static void createJobs(String projectId, String gcsPath) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Set autoPopulateTimespan to true to scan only new content + boolean autoPopulateTimespan = true; + TimespanConfig timespanConfig = + TimespanConfig.newBuilder() + .setEnableAutoPopulationOfTimespanConfig(autoPopulateTimespan) + .build(); + + // Specify the GCS file to be inspected. + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder() + .setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl(gcsPath)) + .build(); + StorageConfig storageConfig = + StorageConfig.newBuilder() + .setCloudStorageOptions(cloudStorageOptions) + .setTimespanConfig(timespanConfig) + .build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("EMAIL_ADDRESS", "PERSON_NAME", "LOCATION", "PHONE_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + // The minimum likelihood required before returning a match: + // See: https://cloud.google.com/dlp/docs/likelihood + Likelihood minLikelihood = Likelihood.UNLIKELY; + + // The maximum number of findings to report (0 = server maximum) + InspectConfig.FindingLimits findingLimits = + InspectConfig.FindingLimits.newBuilder().setMaxFindingsPerItem(100).build(); + + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setIncludeQuote(true) + .setMinLikelihood(minLikelihood) + .setLimits(findingLimits) + .build(); + + // Specify the action that is triggered when the job completes. + Action.PublishSummaryToCscc publishSummaryToCscc = + Action.PublishSummaryToCscc.getDefaultInstance(); + Action action = Action.newBuilder().setPublishSummaryToCscc(publishSummaryToCscc).build(); + + // Configure the inspection job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setInspectConfig(inspectConfig) + .setStorageConfig(storageConfig) + .addActions(action) + .build(); + + // Construct the job creation request to be sent by the client. + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectJob(inspectJobConfig) + .build(); + + // Send the job creation request and process the response. + DlpJob createdDlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + System.out.println("Job created successfully: " + createdDlpJob.getName()); + } + } +} +// [END dlp_create_job] diff --git a/dlp/snippets/src/main/java/dlp/snippets/JobsDelete.java b/dlp/snippets/src/main/java/dlp/snippets/JobsDelete.java new file mode 100644 index 00000000000..c3000d69fd2 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/JobsDelete.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_delete_job] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.DeleteDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJobName; +import java.io.IOException; + +public class JobsDelete { + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String jobId = "your-job-id"; + deleteJobs(projectId, jobId); + } + + // Deletes a DLP Job with the given jobId + public static void deleteJobs(String projectId, String jobId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Construct the complete job name from the projectId and jobId + DlpJobName jobName = DlpJobName.of(projectId, jobId); + + // Construct the job deletion request to be sent by the client. + DeleteDlpJobRequest deleteDlpJobRequest = + DeleteDlpJobRequest.newBuilder().setName(jobName.toString()).build(); + + // Send the job deletion request + dlpServiceClient.deleteDlpJob(deleteDlpJobRequest); + System.out.println("Job deleted successfully."); + } + } +} +// [END dlp_delete_job] diff --git a/dlp/snippets/src/main/java/dlp/snippets/JobsGet.java b/dlp/snippets/src/main/java/dlp/snippets/JobsGet.java new file mode 100644 index 00000000000..9f43eccc93e --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/JobsGet.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_get_job] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.DlpJobName; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import java.io.IOException; + +public class JobsGet { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String jobId = "your-job-id"; + getJobs(projectId, jobId); + } + + // Gets a DLP Job with the given jobId + public static void getJobs(String projectId, String jobId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Construct the complete job name from the projectId and jobId + DlpJobName jobName = DlpJobName.of(projectId, jobId); + + // Construct the get job request to be sent by the client. + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(jobName.toString()).build(); + + // Send the get job request + dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job got successfully."); + } + } +} +// [END dlp_get_job] diff --git a/dlp/snippets/src/main/java/dlp/snippets/JobsList.java b/dlp/snippets/src/main/java/dlp/snippets/JobsList.java new file mode 100644 index 00000000000..892f58e1586 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/JobsList.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_list_jobs] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.DlpJobType; +import com.google.privacy.dlp.v2.ListDlpJobsRequest; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; + +public class JobsList { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + listJobs(projectId); + } + + // Lists DLP jobs + public static void listJobs(String projectId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Construct the request to be sent by the client. + // For more info on filters and job types, + // see https://cloud.google.com/dlp/docs/reference/rest/v2/projects.dlpJobs/list + ListDlpJobsRequest listDlpJobsRequest = + ListDlpJobsRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setFilter("state=DONE") + .setType(DlpJobType.valueOf("INSPECT_JOB")) + .build(); + + // Send the request to list jobs and process the response + DlpServiceClient.ListDlpJobsPagedResponse response = + dlpServiceClient.listDlpJobs(listDlpJobsRequest); + + System.out.println("DLP jobs found:"); + for (DlpJob dlpJob : response.getPage().getValues()) { + System.out.println(dlpJob.getName() + " -- " + dlpJob.getState()); + } + } + } +} +// [END dlp_list_jobs] diff --git a/dlp/snippets/src/main/java/dlp/snippets/QuickStart.java b/dlp/snippets/src/main/java/dlp/snippets/QuickStart.java new file mode 100644 index 00000000000..b4b724deb74 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/QuickStart.java @@ -0,0 +1,112 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_quickstart] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.Finding; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectContentRequest; +import com.google.privacy.dlp.v2.InspectContentResponse; +import com.google.privacy.dlp.v2.InspectResult; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class QuickStart { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + quickstart(projectId); + } + + public static void quickstart(String projectId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + // Configure that content that will be inspected + String text = "His name was Robert Frost"; + ByteContentItem byteContentItem = + ByteContentItem.newBuilder() + .setType(ByteContentItem.BytesType.TEXT_UTF8) + .setData(ByteString.copyFromUtf8(text)) + .build(); + ContentItem contentItem = ContentItem.newBuilder().setByteItem(byteContentItem).build(); + + // The types of information to match: + // See: https://cloud.google.com/dlp/docs/infotypes-reference + List infoTypes = + Stream.of("PERSON_NAME", "US_STATE") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + // The minimum likelihood required before returning a match: + // See: https://cloud.google.com/dlp/docs/likelihood + Likelihood minLikelihood = Likelihood.POSSIBLE; + + // The maximum number of findings to report (0 = server maximum) + InspectConfig.FindingLimits findingLimits = + InspectConfig.FindingLimits.newBuilder().setMaxFindingsPerItem(0).build(); + + // Specify the inspection configuration + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .setLimits(findingLimits) + .setIncludeQuote(true) + .build(); + + // Create the request from previous configs + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectConfig(inspectConfig) + .setItem(contentItem) + .build(); + + // Send the request to the service and receive the results + InspectContentResponse response = dlpServiceClient.inspectContent(request); + + // Process the results + System.out.println("Inspect of text complete: "); + InspectResult result = response.getResult(); + if (result.getFindingsCount() < 0) { + System.out.println("No findings."); + return; + } + System.out.println("Findings: "); + for (Finding finding : result.getFindingsList()) { + System.out.println("\tQuote: " + finding.getQuote()); + System.out.println("\tInfo type: " + finding.getInfoType().getName()); + System.out.println("\tLikelihood: " + finding.getLikelihood()); + } + } + } +} + +// [END dlp_quickstart] diff --git a/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTableWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTableWithFpe.java new file mode 100644 index 00000000000..2838d96dbc9 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTableWithFpe.java @@ -0,0 +1,126 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_reidentify_table_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.ReidentifyContentRequest; +import com.google.privacy.dlp.v2.ReidentifyContentResponse; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class ReIdentifyTableWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + Table tableToReIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("28777").build()) + .build()) + .build(); + reIdentifyTableWithFpe(projectId, tableToReIdentify, kmsKeyName, wrappedAesKey); + } + + public static void reIdentifyTableWithFpe( + String projectId, Table tableToReIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to re-identify. + ContentItem contentItem = ContentItem.newBuilder().setTable(tableToReIdentify).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it. + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how to un-encrypt the previously de-identified information. + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + + // Specify field to be decrypted. + FieldId fieldId = FieldId.newBuilder().setName("Employee ID").build(); + + // Associate the decryption with the specified field. + FieldTransformation fieldTransformation = + FieldTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addFields(fieldId) + .build(); + RecordTransformations transformations = + RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build(); + + DeidentifyConfig reidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build(); + + // Combine configurations into a request for the service. + ReidentifyContentRequest request = + ReidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setReidentifyConfig(reidentifyConfig) + .build(); + + // Send the request and receive response from the service + ReidentifyContentResponse response = dlp.reidentifyContent(request); + + // Print the results + System.out.println("Table after re-identification: " + response.getItem().getValue()); + } + } +} +// [END dlp_reidentify_table_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTextWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTextWithFpe.java new file mode 100644 index 00000000000..1ccdb58ecf2 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyTextWithFpe.java @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_reidentify_text_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.SurrogateType; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReidentifyContentRequest; +import com.google.privacy.dlp.v2.ReidentifyContentResponse; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class ReIdentifyTextWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToReIdentify = "My phone number is PHONE_TOKEN(10):9617256398"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + reIdentifyTextWithFpe(projectId, textToReIdentify, kmsKeyName, wrappedAesKey); + } + + public static void reIdentifyTextWithFpe( + String projectId, String textToReIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to re-identify. + ContentItem contentItem = ContentItem.newBuilder().setValue(textToReIdentify).build(); + + // Specify the type of info the inspection will re-identify. This must use the same custom + // into type that was used as a surrogate during the initial encryption. + InfoType surrogateInfoType = InfoType.newBuilder().setName("PHONE_NUMBER").build(); + + CustomInfoType customInfoType = + CustomInfoType.newBuilder() + .setInfoType(surrogateInfoType) + .setSurrogateType(SurrogateType.getDefaultInstance()) + .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addCustomInfoTypes(customInfoType).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it. + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how to un-encrypt the previously de-identified information. + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .setSurrogateInfoType(surrogateInfoType) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addInfoTypes(surrogateInfoType) + .build(); + InfoTypeTransformations transformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + DeidentifyConfig reidentifyConfig = + DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build(); + + // Combine configurations into a request for the service. + ReidentifyContentRequest request = + ReidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setReidentifyConfig(reidentifyConfig) + .build(); + + // Send the request and receive response from the service + ReidentifyContentResponse response = dlp.reidentifyContent(request); + + // Print the results + System.out.println("Text after re-identification: " + response.getItem().getValue()); + } + } +} +// [END dlp_reidentify_text_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyWithFpe.java b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyWithFpe.java new file mode 100644 index 00000000000..02c436cd2aa --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/ReIdentifyWithFpe.java @@ -0,0 +1,128 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_reidentify_fpe] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.io.BaseEncoding; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoKey; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.CustomInfoType.SurrogateType; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.KmsWrappedCryptoKey; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrimitiveTransformation; +import com.google.privacy.dlp.v2.ReidentifyContentRequest; +import com.google.privacy.dlp.v2.ReidentifyContentResponse; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class ReIdentifyWithFpe { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String textToReIdentify = "My SSN is SSN_TOKEN(9):731997681"; + String kmsKeyName = + "projects/YOUR_PROJECT/" + + "locations/YOUR_KEYRING_REGION/" + + "keyRings/YOUR_KEYRING_NAME/" + + "cryptoKeys/YOUR_KEY_NAME"; + String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; + reIdentifyWithFpe(projectId, textToReIdentify, kmsKeyName, wrappedAesKey); + } + + public static void reIdentifyWithFpe( + String projectId, String textToReIdentify, String kmsKeyName, String wrappedAesKey) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify what content you want the service to re-identify + ContentItem contentItem = ContentItem.newBuilder().setValue(textToReIdentify).build(); + + // Specify the type of info the inspection will re-identify. This must use the same custom + // into type that was used as a surrogate during the initial encryption. + InfoType surrogateInfoType = InfoType.newBuilder().setName("SSN_TOKEN").build(); + + CustomInfoType customInfoType = + CustomInfoType.newBuilder() + .setInfoType(surrogateInfoType) + .setSurrogateType(SurrogateType.getDefaultInstance()) + .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder().addCustomInfoTypes(customInfoType).build(); + + // Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it + KmsWrappedCryptoKey kmsWrappedCryptoKey = + KmsWrappedCryptoKey.newBuilder() + .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey))) + .setCryptoKeyName(kmsKeyName) + .build(); + CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build(); + + // Specify how to un-encrypt the previously de-identified information + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = + CryptoReplaceFfxFpeConfig.newBuilder() + .setCryptoKey(cryptoKey) + // Set of characters in the input text. For more info, see + // https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet + .setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC) + .setSurrogateInfoType(surrogateInfoType) + .build(); + PrimitiveTransformation primitiveTransformation = + PrimitiveTransformation.newBuilder() + .setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig) + .build(); + InfoTypeTransformation infoTypeTransformation = + InfoTypeTransformation.newBuilder() + .setPrimitiveTransformation(primitiveTransformation) + .addInfoTypes(surrogateInfoType) + .build(); + InfoTypeTransformations transformations = + InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build(); + + DeidentifyConfig reidentifyConfig = + DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build(); + + // Combine configurations into a request for the service. + ReidentifyContentRequest request = + ReidentifyContentRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setItem(contentItem) + .setInspectConfig(inspectConfig) + .setReidentifyConfig(reidentifyConfig) + .build(); + + // Send the request and receive response from the service + ReidentifyContentResponse response = dlp.reidentifyContent(request); + + // Print the results + System.out.println("Text after re-identification: " + response.getItem().getValue()); + } + } +} +// [END dlp_reidentify_fpe] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RedactImageFile.java b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFile.java new file mode 100644 index 00000000000..a872ef79b3e --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFile.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_redact_image] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RedactImageRequest; +import com.google.privacy.dlp.v2.RedactImageResponse; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +class RedactImageFile { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/test.png"; + String outputPath = "redacted.png"; + redactImageFile(projectId, inputPath, outputPath); + } + + static void redactImageFile(String projectId, String inputPath, String outputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be inspected. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE).setData(fileBytes).build(); + + // Specify the type of info and likelihood necessary to redact. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : new String[] {"PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(Likelihood.LIKELY) + .build(); + + // Construct the Redact request to be sent by the client. + RedactImageRequest request = + RedactImageRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setByteItem(byteItem) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + RedactImageResponse response = dlp.redactImage(request); + + // Parse the response and process results. + FileOutputStream redacted = new FileOutputStream(outputPath); + redacted.write(response.getRedactedImage().toByteArray()); + redacted.close(); + System.out.println("Redacted image written to " + outputPath); + } + } +} +// [END dlp_redact_image] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllInfoTypes.java new file mode 100644 index 00000000000..c7223fcfdcf --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllInfoTypes.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_redact_image_all_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RedactImageRequest; +import com.google.privacy.dlp.v2.RedactImageResponse; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +class RedactImageFileAllInfoTypes { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/sensitive-data-image.jpeg"; + String outputPath = "sensitive-data-image-redacted.jpeg"; + redactImageFileAllInfoTypes(projectId, inputPath, outputPath); + } + + static void redactImageFileAllInfoTypes(String projectId, String inputPath, String outputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be redacted. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Construct the Redact request to be sent by the client. + // Do not specify the type of info to redact. + RedactImageRequest request = + RedactImageRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setByteItem(byteItem) + .build(); + + // Use the client to send the API request. + RedactImageResponse response = dlp.redactImage(request); + + // Parse the response and process results. + FileOutputStream redacted = new FileOutputStream(outputPath); + redacted.write(response.getRedactedImage().toByteArray()); + redacted.close(); + System.out.println("Redacted image written to " + outputPath); + } + } +} +// [END dlp_redact_image_all_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllText.java b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllText.java new file mode 100644 index 00000000000..c875f2e00db --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileAllText.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_redact_image_all_text] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RedactImageRequest; +import com.google.privacy.dlp.v2.RedactImageRequest.ImageRedactionConfig; +import com.google.privacy.dlp.v2.RedactImageResponse; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +class RedactImageFileAllText { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/sensitive-data-image.jpeg"; + String outputPath = "sensitive-data-image-redacted.jpeg"; + redactImageFileAllText(projectId, inputPath, outputPath); + } + + static void redactImageFileAllText(String projectId, String inputPath, String outputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be redacted. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Enable redaction of all text. + ImageRedactionConfig imageRedactionConfig = + ImageRedactionConfig.newBuilder().setRedactAllText(true).build(); + + // Construct the Redact request to be sent by the client. + // Do not specify the type of info to redact. + RedactImageRequest request = + RedactImageRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setByteItem(byteItem) + .addImageRedactionConfigs(imageRedactionConfig) + .build(); + + // Use the client to send the API request. + RedactImageResponse response = dlp.redactImage(request); + + // Parse the response and process results. + FileOutputStream redacted = new FileOutputStream(outputPath); + redacted.write(response.getRedactedImage().toByteArray()); + redacted.close(); + System.out.println("Redacted image written to " + outputPath); + } + } +} +// [END dlp_redact_image_all_text] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileColoredInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileColoredInfoTypes.java new file mode 100644 index 00000000000..3ed79863cc6 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileColoredInfoTypes.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_redact_image_colored_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.Color; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RedactImageRequest; +import com.google.privacy.dlp.v2.RedactImageRequest.ImageRedactionConfig; +import com.google.privacy.dlp.v2.RedactImageResponse; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +class RedactImageFileColoredInfoTypes { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/test.png"; + String outputPath = "redacted.png"; + redactImageFileColoredInfoTypes(projectId, inputPath, outputPath); + } + + static void redactImageFileColoredInfoTypes(String projectId, String inputPath, String outputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be redacted. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Define types of info to redact associate each one with a different color. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + ImageRedactionConfig ssnRedactionConfig = + ImageRedactionConfig.newBuilder() + .setInfoType(InfoType.newBuilder().setName("US_SOCIAL_SECURITY_NUMBER").build()) + .setRedactionColor(Color.newBuilder().setRed(.3f).setGreen(.1f).setBlue(.6f).build()) + .build(); + ImageRedactionConfig emailRedactionConfig = + ImageRedactionConfig.newBuilder() + .setInfoType(InfoType.newBuilder().setName("EMAIL_ADDRESS").build()) + .setRedactionColor(Color.newBuilder().setRed(.5f).setGreen(.5f).setBlue(1).build()) + .build(); + ImageRedactionConfig phoneRedactionConfig = + ImageRedactionConfig.newBuilder() + .setInfoType(InfoType.newBuilder().setName("PHONE_NUMBER").build()) + .setRedactionColor(Color.newBuilder().setRed(1).setGreen(0).setBlue(.6f).build()) + .build(); + + // Create collection of all redact configurations. + List imageRedactionConfigs = + Arrays.asList(ssnRedactionConfig, emailRedactionConfig, phoneRedactionConfig); + + // List types of info to search for. + InspectConfig config = + InspectConfig.newBuilder() + .addAllInfoTypes( + imageRedactionConfigs.stream() + .map(ImageRedactionConfig::getInfoType) + .collect(Collectors.toList())) + .build(); + + // Construct the Redact request to be sent by the client. + RedactImageRequest request = + RedactImageRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setByteItem(byteItem) + .addAllImageRedactionConfigs(imageRedactionConfigs) + .setInspectConfig(config) + .build(); + + // Use the client to send the API request. + RedactImageResponse response = dlp.redactImage(request); + + // Parse the response and process results. + FileOutputStream redacted = new FileOutputStream(outputPath); + redacted.write(response.getRedactedImage().toByteArray()); + redacted.close(); + System.out.println("Redacted image written to " + outputPath); + } + } +} +// [END dlp_redact_image_colored_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileListedInfoTypes.java b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileListedInfoTypes.java new file mode 100644 index 00000000000..e7b909b3b64 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RedactImageFileListedInfoTypes.java @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_redact_image_listed_infotypes] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.ByteContentItem; +import com.google.privacy.dlp.v2.ByteContentItem.BytesType; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.RedactImageRequest; +import com.google.privacy.dlp.v2.RedactImageRequest.ImageRedactionConfig; +import com.google.privacy.dlp.v2.RedactImageResponse; +import com.google.protobuf.ByteString; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +class RedactImageFileListedInfoTypes { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String inputPath = "src/test/resources/sensitive-data-image.jpeg"; + String outputPath = "sensitive-data-image-redacted.jpeg"; + redactImageFileListedInfoTypes(projectId, inputPath, outputPath); + } + + static void redactImageFileListedInfoTypes(String projectId, String inputPath, String outputPath) + throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlp = DlpServiceClient.create()) { + // Specify the content to be redacted. + ByteString fileBytes = ByteString.readFrom(new FileInputStream(inputPath)); + ByteContentItem byteItem = + ByteContentItem.newBuilder().setType(BytesType.IMAGE_JPEG).setData(fileBytes).build(); + + // Specify the types of info necessary to redact. + List infoTypes = new ArrayList<>(); + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + for (String typeName : + new String[] {"US_SOCIAL_SECURITY_NUMBER", "EMAIL_ADDRESS", "PHONE_NUMBER"}) { + infoTypes.add(InfoType.newBuilder().setName(typeName).build()); + } + InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).build(); + + // Prepare redaction configs. + List imageRedactionConfigs = + infoTypes.stream() + .map(infoType -> ImageRedactionConfig.newBuilder().setInfoType(infoType).build()) + .collect(Collectors.toList()); + + // Construct the Redact request to be sent by the client. + RedactImageRequest request = + RedactImageRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setByteItem(byteItem) + .addAllImageRedactionConfigs(imageRedactionConfigs) + .setInspectConfig(inspectConfig) + .build(); + + // Use the client to send the API request. + RedactImageResponse response = dlp.redactImage(request); + + // Parse the response and process results. + FileOutputStream redacted = new FileOutputStream(outputPath); + redacted.write(response.getRedactedImage().toByteArray()); + redacted.close(); + System.out.println("Redacted image written to " + outputPath); + } + } +} +// [END dlp_redact_image_listed_infotypes] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisCategoricalStats.java b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisCategoricalStats.java new file mode 100644 index 00000000000..a290281a52f --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisCategoricalStats.java @@ -0,0 +1,181 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_categorical_stats] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.Action.PublishToPubSub; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.CategoricalStatsResult; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.CategoricalStatsResult.CategoricalStatsHistogramBucket; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrivacyMetric; +import com.google.privacy.dlp.v2.PrivacyMetric.CategoricalStatsConfig; +import com.google.privacy.dlp.v2.RiskAnalysisJobConfig; +import com.google.privacy.dlp.v2.ValueFrequency; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +class RiskAnalysisCategoricalStats { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datasetId = "your-bigquery-dataset-id"; + String tableId = "your-bigquery-table-id"; + String topicId = "pub-sub-topic"; + String subscriptionId = "pub-sub-subscription"; + categoricalStatsAnalysis(projectId, datasetId, tableId, topicId, subscriptionId); + } + + public static void categoricalStatsAnalysis( + String projectId, String datasetId, String tableId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + // Specify the BigQuery table to analyze + BigQueryTable bigQueryTable = + BigQueryTable.newBuilder() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId) + .build(); + + // The name of the column to analyze, which doesn't need to contain numerical data + String columnName = "Mystery"; + + // Configure the privacy metric for the job + FieldId fieldId = FieldId.newBuilder().setName(columnName).build(); + CategoricalStatsConfig categoricalStatsConfig = + CategoricalStatsConfig.newBuilder().setField(fieldId).build(); + PrivacyMetric privacyMetric = + PrivacyMetric.newBuilder().setCategoricalStatsConfig(categoricalStatsConfig).build(); + + // Create action to publish job status notifications over Google Cloud Pub/Sub + ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId); + PublishToPubSub publishToPubSub = + PublishToPubSub.newBuilder().setTopic(topicName.toString()).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the risk analysis job to perform + RiskAnalysisJobConfig riskAnalysisJobConfig = + RiskAnalysisJobConfig.newBuilder() + .setSourceTable(bigQueryTable) + .setPrivacyMetric(privacyMetric) + .addActions(action) + .build(); + + // Build the job creation request to be sent by the client + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setRiskJob(riskAnalysisJobConfig) + .build(); + + // Send the request to the API using the client + DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Build a request to get the completed job + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + + // Retrieve completed job status + DlpJob completedJob = dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + + // Get the result and parse through and process the information + CategoricalStatsResult result = completedJob.getRiskDetails().getCategoricalStatsResult(); + List histogramBucketList = + result.getValueFrequencyHistogramBucketsList(); + + for (CategoricalStatsHistogramBucket bucket : histogramBucketList) { + long mostCommonFrequency = bucket.getValueFrequencyUpperBound(); + System.out.printf("Most common value occurs %d time(s).\n", mostCommonFrequency); + + long leastCommonFrequency = bucket.getValueFrequencyLowerBound(); + System.out.printf("Least common value occurs %d time(s).\n", leastCommonFrequency); + + for (ValueFrequency valueFrequency : bucket.getBucketValuesList()) { + System.out.printf( + "Value %s occurs %d time(s).\n", + valueFrequency.getValue().toString(), valueFrequency.getCount()); + } + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} + +// [END dlp_categorical_stats] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKAnonymity.java b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKAnonymity.java new file mode 100644 index 00000000000..f9c29a0e932 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKAnonymity.java @@ -0,0 +1,189 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_k_anonymity] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.Action.PublishToPubSub; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult.KAnonymityEquivalenceClass; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult.KAnonymityHistogramBucket; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrivacyMetric; +import com.google.privacy.dlp.v2.PrivacyMetric.KAnonymityConfig; +import com.google.privacy.dlp.v2.RiskAnalysisJobConfig; +import com.google.privacy.dlp.v2.Value; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") +class RiskAnalysisKAnonymity { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datasetId = "your-bigquery-dataset-id"; + String tableId = "your-bigquery-table-id"; + String topicId = "pub-sub-topic"; + String subscriptionId = "pub-sub-subscription"; + calculateKAnonymity(projectId, datasetId, tableId, topicId, subscriptionId); + } + + public static void calculateKAnonymity( + String projectId, String datasetId, String tableId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Specify the BigQuery table to analyze + BigQueryTable bigQueryTable = + BigQueryTable.newBuilder() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId) + .build(); + + // These values represent the column names of quasi-identifiers to analyze + List quasiIds = Arrays.asList("Age", "Mystery"); + + // Configure the privacy metric for the job + List quasiIdFields = + quasiIds.stream() + .map(columnName -> FieldId.newBuilder().setName(columnName).build()) + .collect(Collectors.toList()); + KAnonymityConfig kanonymityConfig = + KAnonymityConfig.newBuilder().addAllQuasiIds(quasiIdFields).build(); + PrivacyMetric privacyMetric = + PrivacyMetric.newBuilder().setKAnonymityConfig(kanonymityConfig).build(); + + // Create action to publish job status notifications over Google Cloud Pub/Sub + ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId); + PublishToPubSub publishToPubSub = + PublishToPubSub.newBuilder().setTopic(topicName.toString()).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the risk analysis job to perform + RiskAnalysisJobConfig riskAnalysisJobConfig = + RiskAnalysisJobConfig.newBuilder() + .setSourceTable(bigQueryTable) + .setPrivacyMetric(privacyMetric) + .addActions(action) + .build(); + + // Build the request to be sent by the client + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setRiskJob(riskAnalysisJobConfig) + .build(); + + // Send the request to the API using the client + DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Build a request to get the completed job + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + + // Retrieve completed job status + DlpJob completedJob = dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + + // Get the result and parse through and process the information + KAnonymityResult kanonymityResult = completedJob.getRiskDetails().getKAnonymityResult(); + List histogramBucketList = + kanonymityResult.getEquivalenceClassHistogramBucketsList(); + for (KAnonymityHistogramBucket result : histogramBucketList) { + System.out.printf( + "Bucket size range: [%d, %d]\n", + result.getEquivalenceClassSizeLowerBound(), result.getEquivalenceClassSizeUpperBound()); + + for (KAnonymityEquivalenceClass bucket : result.getBucketValuesList()) { + List quasiIdValues = + bucket.getQuasiIdsValuesList().stream() + .map(Value::toString) + .collect(Collectors.toList()); + + System.out.println("\tQuasi-ID values: " + String.join(", ", quasiIdValues)); + System.out.println("\tClass size: " + bucket.getEquivalenceClassSize()); + } + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_k_anonymity] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKMap.java b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKMap.java new file mode 100644 index 00000000000..9dae52ac67a --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisKMap.java @@ -0,0 +1,219 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_k_map] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.Action.PublishToPubSub; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult.KMapEstimationHistogramBucket; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult.KMapEstimationQuasiIdValues; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrivacyMetric; +import com.google.privacy.dlp.v2.PrivacyMetric.KMapEstimationConfig; +import com.google.privacy.dlp.v2.PrivacyMetric.KMapEstimationConfig.TaggedField; +import com.google.privacy.dlp.v2.RiskAnalysisJobConfig; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") +class RiskAnalysisKMap { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datasetId = "your-bigquery-dataset-id"; + String tableId = "your-bigquery-table-id"; + String topicId = "pub-sub-topic"; + String subscriptionId = "pub-sub-subscription"; + calculateKMap(projectId, datasetId, tableId, topicId, subscriptionId); + } + + public static void calculateKMap( + String projectId, String datasetId, String tableId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + // Specify the BigQuery table to analyze + BigQueryTable bigQueryTable = + BigQueryTable.newBuilder() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId) + .build(); + + // These values represent the column names of quasi-identifiers to analyze + List quasiIds = Arrays.asList("Age", "Gender"); + + // These values represent the info types corresponding to the quasi-identifiers above + List infoTypeNames = Arrays.asList("AGE", "GENDER"); + + // Tag each of the quasiId column names with its corresponding infoType + List infoTypes = + infoTypeNames.stream() + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + if (quasiIds.size() != infoTypes.size()) { + throw new IllegalArgumentException("The numbers of quasi-IDs and infoTypes must be equal!"); + } + + List taggedFields = new ArrayList(); + for (int i = 0; i < quasiIds.size(); i++) { + TaggedField taggedField = + TaggedField.newBuilder() + .setField(FieldId.newBuilder().setName(quasiIds.get(i)).build()) + .setInfoType(infoTypes.get(i)) + .build(); + taggedFields.add(taggedField); + } + + // The k-map distribution region can be specified by any ISO-3166-1 region code. + String regionCode = "US"; + + // Configure the privacy metric for the job + KMapEstimationConfig kmapConfig = + KMapEstimationConfig.newBuilder() + .addAllQuasiIds(taggedFields) + .setRegionCode(regionCode) + .build(); + PrivacyMetric privacyMetric = + PrivacyMetric.newBuilder().setKMapEstimationConfig(kmapConfig).build(); + + // Create action to publish job status notifications over Google Cloud Pub/Sub + ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId); + PublishToPubSub publishToPubSub = + PublishToPubSub.newBuilder().setTopic(topicName.toString()).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the risk analysis job to perform + RiskAnalysisJobConfig riskAnalysisJobConfig = + RiskAnalysisJobConfig.newBuilder() + .setSourceTable(bigQueryTable) + .setPrivacyMetric(privacyMetric) + .addActions(action) + .build(); + + // Build the request to be sent by the client + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setRiskJob(riskAnalysisJobConfig) + .build(); + + // Send the request to the API using the client + DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Build a request to get the completed job + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + + // Retrieve completed job status + DlpJob completedJob = dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + + // Get the result and parse through and process the information + KMapEstimationResult kmapResult = completedJob.getRiskDetails().getKMapEstimationResult(); + + for (KMapEstimationHistogramBucket result : kmapResult.getKMapEstimationHistogramList()) { + System.out.printf( + "\tAnonymity range: [%d, %d]\n", result.getMinAnonymity(), result.getMaxAnonymity()); + System.out.printf("\tSize: %d\n", result.getBucketSize()); + + for (KMapEstimationQuasiIdValues valueBucket : result.getBucketValuesList()) { + List quasiIdValues = + valueBucket.getQuasiIdsValuesList().stream() + .map( + value -> { + String s = value.toString(); + return s.substring(s.indexOf(':') + 1).trim(); + }) + .collect(Collectors.toList()); + + System.out.printf("\tValues: {%s}\n", String.join(", ", quasiIdValues)); + System.out.printf( + "\tEstimated k-map anonymity: %d\n", valueBucket.getEstimatedAnonymity()); + } + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_k_map] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisLDiversity.java b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisLDiversity.java new file mode 100644 index 00000000000..2650668574b --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisLDiversity.java @@ -0,0 +1,212 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_l_diversity] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.dlp.v2.DlpServiceSettings; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.Action.PublishToPubSub; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityEquivalenceClass; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityHistogramBucket; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrivacyMetric; +import com.google.privacy.dlp.v2.PrivacyMetric.LDiversityConfig; +import com.google.privacy.dlp.v2.RiskAnalysisJobConfig; +import com.google.privacy.dlp.v2.Value; +import com.google.privacy.dlp.v2.ValueFrequency; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.threeten.bp.Duration; + +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") +class RiskAnalysisLDiversity { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datasetId = "your-bigquery-dataset-id"; + String tableId = "your-bigquery-table-id"; + String topicId = "pub-sub-topic"; + String subscriptionId = "pub-sub-subscription"; + calculateLDiversity(projectId, datasetId, tableId, topicId, subscriptionId); + } + + public static void calculateLDiversity( + String projectId, String datasetId, String tableId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + DlpServiceSettings.Builder dlpServiceSettingsBuilder = DlpServiceSettings.newBuilder(); + dlpServiceSettingsBuilder + .getDlpJobSettings() + .setRetrySettings( + dlpServiceSettingsBuilder + .getDlpJobSettings() + .getRetrySettings() + .toBuilder() + .setTotalTimeout(Duration.ofSeconds(600)) + .build()); + try (DlpServiceClient dlpServiceClient = + DlpServiceClient.create(dlpServiceSettingsBuilder.build())) { + // Specify the BigQuery table to analyze + BigQueryTable bigQueryTable = + BigQueryTable.newBuilder() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId) + .build(); + + // These values represent the column names of quasi-identifiers to analyze + List quasiIds = Arrays.asList("Age", "Mystery"); + + // This value represents the column name to compare the quasi-identifiers against + String sensitiveAttribute = "Name"; + + // Configure the privacy metric for the job + FieldId sensitiveAttributeField = FieldId.newBuilder().setName(sensitiveAttribute).build(); + List quasiIdFields = + quasiIds.stream() + .map(columnName -> FieldId.newBuilder().setName(columnName).build()) + .collect(Collectors.toList()); + LDiversityConfig ldiversityConfig = + LDiversityConfig.newBuilder() + .addAllQuasiIds(quasiIdFields) + .setSensitiveAttribute(sensitiveAttributeField) + .build(); + PrivacyMetric privacyMetric = + PrivacyMetric.newBuilder().setLDiversityConfig(ldiversityConfig).build(); + + // Create action to publish job status notifications over Google Cloud Pub/ + ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId); + PublishToPubSub publishToPubSub = + PublishToPubSub.newBuilder().setTopic(topicName.toString()).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the risk analysis job to perform + RiskAnalysisJobConfig riskAnalysisJobConfig = + RiskAnalysisJobConfig.newBuilder() + .setSourceTable(bigQueryTable) + .setPrivacyMetric(privacyMetric) + .addActions(action) + .build(); + + // Build the request to be sent by the client + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setRiskJob(riskAnalysisJobConfig) + .build(); + + // Send the request to the API using the client + DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Build a request to get the completed job + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + + // Retrieve completed job status + DlpJob completedJob = dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + + // Get the result and parse through and process the information + LDiversityResult ldiversityResult = completedJob.getRiskDetails().getLDiversityResult(); + List histogramBucketList = + ldiversityResult.getSensitiveValueFrequencyHistogramBucketsList(); + for (LDiversityHistogramBucket result : histogramBucketList) { + for (LDiversityEquivalenceClass bucket : result.getBucketValuesList()) { + List quasiIdValues = + bucket.getQuasiIdsValuesList().stream() + .map(Value::toString) + .collect(Collectors.toList()); + + System.out.println("\tQuasi-ID values: " + String.join(", ", quasiIdValues)); + System.out.println("\tClass size: " + bucket.getEquivalenceClassSize()); + + for (ValueFrequency valueFrequency : bucket.getTopSensitiveValuesList()) { + System.out.printf( + "\t\tSensitive value %s occurs %d time(s).\n", + valueFrequency.getValue().toString(), valueFrequency.getCount()); + } + } + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_l_diversity] diff --git a/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisNumericalStats.java b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisNumericalStats.java new file mode 100644 index 00000000000..b54f551eff6 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/RiskAnalysisNumericalStats.java @@ -0,0 +1,177 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_numerical_stats] + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.AckReplyConsumer; +import com.google.cloud.pubsub.v1.MessageReceiver; +import com.google.cloud.pubsub.v1.Subscriber; +import com.google.privacy.dlp.v2.Action; +import com.google.privacy.dlp.v2.Action.PublishToPubSub; +import com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.NumericalStatsResult; +import com.google.privacy.dlp.v2.BigQueryTable; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.GetDlpJobRequest; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.PrivacyMetric; +import com.google.privacy.dlp.v2.PrivacyMetric.NumericalStatsConfig; +import com.google.privacy.dlp.v2.RiskAnalysisJobConfig; +import com.google.privacy.dlp.v2.Value; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +class RiskAnalysisNumericalStats { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String datasetId = "your-bigquery-dataset-id"; + String tableId = "your-bigquery-table-id"; + String topicId = "pub-sub-topic"; + String subscriptionId = "pub-sub-subscription"; + numericalStatsAnalysis(projectId, datasetId, tableId, topicId, subscriptionId); + } + + public static void numericalStatsAnalysis( + String projectId, String datasetId, String tableId, String topicId, String subscriptionId) + throws ExecutionException, InterruptedException, IOException { + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Specify the BigQuery table to analyze + BigQueryTable bigQueryTable = + BigQueryTable.newBuilder() + .setTableId(tableId) + .setDatasetId(datasetId) + .setProjectId(projectId) + .build(); + + // This represents the name of the column to analyze, which must contain numerical data + String columnName = "Age"; + + // Configure the privacy metric for the job + FieldId fieldId = FieldId.newBuilder().setName(columnName).build(); + NumericalStatsConfig numericalStatsConfig = + NumericalStatsConfig.newBuilder().setField(fieldId).build(); + PrivacyMetric privacyMetric = + PrivacyMetric.newBuilder().setNumericalStatsConfig(numericalStatsConfig).build(); + + // Create action to publish job status notifications over Google Cloud Pub/Sub + ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId); + PublishToPubSub publishToPubSub = + PublishToPubSub.newBuilder().setTopic(topicName.toString()).build(); + Action action = Action.newBuilder().setPubSub(publishToPubSub).build(); + + // Configure the risk analysis job to perform + RiskAnalysisJobConfig riskAnalysisJobConfig = + RiskAnalysisJobConfig.newBuilder() + .setSourceTable(bigQueryTable) + .setPrivacyMetric(privacyMetric) + .addActions(action) + .build(); + + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setRiskJob(riskAnalysisJobConfig) + .build(); + + // Send the request to the API using the client + DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest); + + // Set up a Pub/Sub subscriber to listen on the job completion status + final SettableApiFuture done = SettableApiFuture.create(); + + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(projectId, subscriptionId); + + MessageReceiver messageHandler = + (PubsubMessage pubsubMessage, AckReplyConsumer ackReplyConsumer) -> { + handleMessage(dlpJob, done, pubsubMessage, ackReplyConsumer); + }; + Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageHandler).build(); + subscriber.startAsync(); + + // Wait for job completion semi-synchronously + // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions + try { + done.get(15, TimeUnit.MINUTES); + } catch (TimeoutException e) { + System.out.println("Job was not completed after 15 minutes."); + return; + } finally { + subscriber.stopAsync(); + subscriber.awaitTerminated(); + } + + // Build a request to get the completed job + GetDlpJobRequest getDlpJobRequest = + GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build(); + + // Retrieve completed job status + DlpJob completedJob = dlpServiceClient.getDlpJob(getDlpJobRequest); + System.out.println("Job status: " + completedJob.getState()); + System.out.println("Job name: " + dlpJob.getName()); + + // Get the result and parse through and process the information + NumericalStatsResult result = completedJob.getRiskDetails().getNumericalStatsResult(); + + System.out.printf( + "Value range : [%.3f, %.3f]\n", + result.getMinValue().getFloatValue(), result.getMaxValue().getFloatValue()); + + int percent = 1; + Double lastValue = null; + for (Value quantileValue : result.getQuantileValuesList()) { + Double currentValue = quantileValue.getFloatValue(); + if (lastValue == null || !lastValue.equals(currentValue)) { + System.out.printf("Value at %s %% quantile : %.3f", percent, currentValue); + } + lastValue = currentValue; + } + } + } + + // handleMessage injects the job and settableFuture into the message reciever interface + private static void handleMessage( + DlpJob job, + SettableApiFuture done, + PubsubMessage pubsubMessage, + AckReplyConsumer ackReplyConsumer) { + String messageAttribute = pubsubMessage.getAttributesMap().get("DlpJobName"); + if (job.getName().equals(messageAttribute)) { + done.set(true); + ackReplyConsumer.ack(); + } else { + ackReplyConsumer.nack(); + } + } +} +// [END dlp_numerical_stats] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TemplatesCreate.java b/dlp/snippets/src/main/java/dlp/snippets/TemplatesCreate.java new file mode 100644 index 00000000000..fe8ef939bb1 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TemplatesCreate.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_create_inspect_template] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.CreateInspectTemplateRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectTemplate; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +class TemplatesCreate { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + createInspectTemplate(projectId); + } + + // Creates a template to persist configuration information + public static void createInspectTemplate(String projectId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + // Construct the inspection configuration for the template + InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).build(); + + // Optionally set a display name and a description for the template + String displayName = "Inspection Config Template"; + String description = "Save configuration for future inspection jobs"; + + // Build the template + InspectTemplate inspectTemplate = + InspectTemplate.newBuilder() + .setInspectConfig(inspectConfig) + .setDisplayName(displayName) + .setDescription(description) + .build(); + + // Create the request to be sent by the client + CreateInspectTemplateRequest createInspectTemplateRequest = + CreateInspectTemplateRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setInspectTemplate(inspectTemplate) + .build(); + + // Send the request to the API and process the response + InspectTemplate response = + dlpServiceClient.createInspectTemplate(createInspectTemplateRequest); + System.out.printf("Template created: %s", response.getName()); + } + } +} +// [END dlp_create_inspect_template] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TemplatesDelete.java b/dlp/snippets/src/main/java/dlp/snippets/TemplatesDelete.java new file mode 100644 index 00000000000..f22c362ff9f --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TemplatesDelete.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_delete_inspect_template] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.DeleteInspectTemplateRequest; +import java.io.IOException; + +class TemplatesDelete { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String templateId = "your-template-id"; + deleteInspectTemplate(projectId, templateId); + } + + // Delete an existing template + public static void deleteInspectTemplate(String projectId, String templateId) throws IOException { + // Construct the template name to be deleted + String templateName = String.format("projects/%s/inspectTemplates/%s", projectId, templateId); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Create delete template request to be sent by the client + DeleteInspectTemplateRequest request = + DeleteInspectTemplateRequest.newBuilder().setName(templateName).build(); + + // Send the request with the client + dlpServiceClient.deleteInspectTemplate(request); + System.out.printf("Deleted template: %s\n", templateName); + } + } +} +// [END dlp_delete_inspect_template] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TemplatesList.java b/dlp/snippets/src/main/java/dlp/snippets/TemplatesList.java new file mode 100644 index 00000000000..729782f529b --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TemplatesList.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_list_inspect_templates] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.dlp.v2.DlpServiceClient.ListInspectTemplatesPagedResponse; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectTemplate; +import com.google.privacy.dlp.v2.ListInspectTemplatesRequest; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; + +class TemplatesList { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + listInspectTemplates(projectId); + } + + // Lists all templates associated with a given project + public static void listInspectTemplates(String projectId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Create the request to be sent by the client + ListInspectTemplatesRequest request = + ListInspectTemplatesRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setPageSize(1) + .build(); + + // Send the request + ListInspectTemplatesPagedResponse response = dlpServiceClient.listInspectTemplates(request); + + // Parse through and process the response + System.out.println("Templates found:"); + for (InspectTemplate template : response.getPage().getResponse().getInspectTemplatesList()) { + System.out.printf("Template name: %s\n", template.getName()); + if (template.getDisplayName() != null) { + System.out.printf("\tDisplay name: %s \n", template.getDisplayName()); + System.out.printf("\tCreate time: %s \n", template.getCreateTime()); + System.out.printf("\tUpdate time: %s \n", template.getUpdateTime()); + + // print inspection config + InspectConfig inspectConfig = template.getInspectConfig(); + for (InfoType infoType : inspectConfig.getInfoTypesList()) { + System.out.printf("\tInfoType: %s\n", infoType.getName()); + } + System.out.printf("\tMin likelihood: %s\n", inspectConfig.getMinLikelihood()); + System.out.printf("\tLimits: %s\n", inspectConfig.getLimits().getMaxFindingsPerRequest()); + } + } + } + } +} +// [END dlp_list_inspect_templates] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TriggersCreate.java b/dlp/snippets/src/main/java/dlp/snippets/TriggersCreate.java new file mode 100644 index 00000000000..de0dd80dfd8 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TriggersCreate.java @@ -0,0 +1,123 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_create_trigger] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CreateJobTriggerRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.JobTrigger; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.Schedule; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.privacy.dlp.v2.StorageConfig.TimespanConfig; +import com.google.protobuf.Duration; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TriggersCreate { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String gcsPath = "gs://" + "your-bucket-name" + "path/to/file.txt"; + createTrigger(projectId, gcsPath); + } + + public static void createTrigger(String projectId, String gcsPath) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Set autoPopulateTimespan to true to scan only new content + boolean autoPopulateTimespan = true; + TimespanConfig timespanConfig = + TimespanConfig.newBuilder() + .setEnableAutoPopulationOfTimespanConfig(autoPopulateTimespan) + .build(); + + // Specify the GCS file to be inspected. + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder() + .setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl(gcsPath)) + .build(); + StorageConfig storageConfig = + StorageConfig.newBuilder() + .setCloudStorageOptions(cloudStorageOptions) + .setTimespanConfig(timespanConfig) + .build(); + + // Specify the type of info the inspection will look for. + // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).build(); + + // Configure the inspection job we want the service to perform. + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setInspectConfig(inspectConfig) + .setStorageConfig(storageConfig) + .build(); + + // Set scanPeriod to the number of days between scans (minimum: 1 day) + int scanPeriod = 1; + + // Optionally set a display name of max 100 chars and a description of max 250 chars + String displayName = "Daily Scan"; + String description = "A daily inspection for personally identifiable information."; + + // Schedule scan of GCS bucket every scanPeriod number of days (minimum = 1 day) + Duration duration = Duration.newBuilder().setSeconds(scanPeriod * 24 * 3600).build(); + Schedule schedule = Schedule.newBuilder().setRecurrencePeriodDuration(duration).build(); + JobTrigger.Trigger trigger = JobTrigger.Trigger.newBuilder().setSchedule(schedule).build(); + JobTrigger jobTrigger = + JobTrigger.newBuilder() + .setInspectJob(inspectJobConfig) + .setDisplayName(displayName) + .setDescription(description) + .setStatus(JobTrigger.Status.HEALTHY) + .addTriggers(trigger) + .build(); + + // Create scan request to be sent by client + CreateJobTriggerRequest createJobTriggerRequest = + CreateJobTriggerRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .setJobTrigger(jobTrigger) + .build(); + + // Send the scan request and process the response + JobTrigger createdJobTrigger = dlpServiceClient.createJobTrigger(createJobTriggerRequest); + + System.out.println("Created Trigger: " + createdJobTrigger.getName()); + System.out.println("Display Name: " + createdJobTrigger.getDisplayName()); + System.out.println("Description: " + createdJobTrigger.getDescription()); + } + } +} +// [END dlp_create_trigger] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TriggersDelete.java b/dlp/snippets/src/main/java/dlp/snippets/TriggersDelete.java new file mode 100644 index 00000000000..539f2537dbc --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TriggersDelete.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_delete_trigger] +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.DeleteJobTriggerRequest; +import com.google.privacy.dlp.v2.ProjectJobTriggerName; +import java.io.IOException; + +class TriggersDelete { + + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + String triggerId = "your-trigger-id"; + deleteTrigger(projectId, triggerId); + } + + public static void deleteTrigger(String projectId, String triggerId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + // Get the full trigger name from the given triggerId and ProjectId + ProjectJobTriggerName triggerName = ProjectJobTriggerName.of(projectId, triggerId); + + // Construct the trigger deletion request to be sent by the client + DeleteJobTriggerRequest deleteJobTriggerRequest = + DeleteJobTriggerRequest.newBuilder().setName(triggerName.toString()).build(); + + // Send the trigger deletion request + dlpServiceClient.deleteJobTrigger(deleteJobTriggerRequest); + System.out.println("Trigger deleted: " + triggerName.toString()); + } + } +} +// [END dlp_delete_trigger] diff --git a/dlp/snippets/src/main/java/dlp/snippets/TriggersList.java b/dlp/snippets/src/main/java/dlp/snippets/TriggersList.java new file mode 100644 index 00000000000..d52146399d7 --- /dev/null +++ b/dlp/snippets/src/main/java/dlp/snippets/TriggersList.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +// [START dlp_list_triggers] + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.privacy.dlp.v2.JobTrigger; +import com.google.privacy.dlp.v2.ListJobTriggersRequest; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; + +class TriggersList { + public static void main(String[] args) throws Exception { + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; + listTriggers(projectId); + } + + public static void listTriggers(String projectId) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + // Build the request to be sent by the client + ListJobTriggersRequest listJobTriggersRequest = + ListJobTriggersRequest.newBuilder() + .setParent(LocationName.of(projectId, "global").toString()) + .build(); + + // Use the client to send the API request. + DlpServiceClient.ListJobTriggersPagedResponse response = + dlpServiceClient.listJobTriggers(listJobTriggersRequest); + + // Parse the response and process the results + System.out.println("DLP triggers found:"); + for (JobTrigger trigger : response.getPage().getValues()) { + System.out.println("Trigger: " + trigger.getName()); + System.out.println("\tCreated: " + trigger.getCreateTime()); + System.out.println("\tUpdated: " + trigger.getUpdateTime()); + if (trigger.getDisplayName() != null) { + System.out.println("\tDisplay name: " + trigger.getDisplayName()); + } + if (trigger.getDescription() != null) { + System.out.println("\tDescription: " + trigger.getDescription()); + } + System.out.println("\tStatus: " + trigger.getStatus()); + System.out.println("\tError count: " + trigger.getErrorsCount()); + } + ; + } + } +} +// [END dlp_list_triggers] diff --git a/dlp/snippets/src/test/java/dlp/snippets/DeIdentificationTests.java b/dlp/snippets/src/test/java/dlp/snippets/DeIdentificationTests.java new file mode 100644 index 00000000000..e691444ec81 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/DeIdentificationTests.java @@ -0,0 +1,544 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.collect.ImmutableList; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DeIdentificationTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of( + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "DLP_DEID_WRAPPED_KEY", + "DLP_DEID_KEY_NAME"); + } + + @Test + public void testDeIdentifyWithMasking() throws IOException { + DeIdentifyWithMasking.deIdentifyWithMasking(PROJECT_ID, "My SSN is 372819127"); + + String output = bout.toString(); + assertThat(output).contains("Text after masking:"); + } + + @Test + public void testDeIdentifyWithFpe() throws IOException { + DeIdentifyWithFpe.deIdentifyWithFpe(PROJECT_ID, "My SSN is 372819127", kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Text after format-preserving encryption:"); + } + + @Test + public void testReIdentifyWithFpe() throws IOException { + ReIdentifyWithFpe.reIdentifyWithFpe( + PROJECT_ID, "My SSN is SSN_TOKEN(9):731997681", kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Text after re-identification:"); + } + + @Test + public void testDeIdentifyTextWithFpe() throws IOException { + DeIdentifyTextWithFpe.deIdentifyTextWithFpe( + PROJECT_ID, "My phone number is 4359916732", kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Text after format-preserving encryption: "); + } + + @Test + public void testReIdentifyTextWithFpe() throws IOException { + ReIdentifyTextWithFpe.reIdentifyTextWithFpe( + PROJECT_ID, "My phone number is PHONE_TOKEN(10):9617256398", kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Text after re-identification: "); + } + + @Test + public void testDeIdentifyTableWithFpe() throws IOException { + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) + .addHeaders(FieldId.newBuilder().setName("Date").build()) + .addHeaders(FieldId.newBuilder().setName("Compensation").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("11111").build()) + .addValues(Value.newBuilder().setStringValue("2015").build()) + .addValues(Value.newBuilder().setStringValue("$10").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("11111").build()) + .addValues(Value.newBuilder().setStringValue("2016").build()) + .addValues(Value.newBuilder().setStringValue("$20").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22222").build()) + .addValues(Value.newBuilder().setStringValue("2016").build()) + .addValues(Value.newBuilder().setStringValue("$15").build()) + .build()) + .build(); + + DeIdentifyTableWithFpe.deIdentifyTableWithFpe( + PROJECT_ID, tableToDeIdentify, kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Table after format-preserving encryption:"); + } + + @Test + public void testReIdentifyTableWithFpe() throws IOException { + Table tableToReIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("28777").build()) + .build()) + .build(); + + ReIdentifyTableWithFpe.reIdentifyTableWithFpe( + PROJECT_ID, tableToReIdentify, kmsKeyName, wrappedKey); + + String output = bout.toString(); + assertThat(output).contains("Table after re-identification:"); + } + + @Test + public void testDeIdentifyTableBucketing() throws IOException { + // Transform a column based on the value of another column + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + Table expectedTable = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("90:100").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("20:30").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("70:80").build()) + .build()) + .build(); + + Table table = DeIdentifyTableBucketing.deIdentifyTableBucketing(PROJECT_ID, tableToDeIdentify); + + String output = bout.toString(); + assertThat(output).contains("Table after de-identification:"); + assertThat(table).isEqualTo(expectedTable); + } + + @Test + public void testDeIdentifyTableConditionMasking() throws IOException { + // Transform a column based on the value of another column + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + Table expectedTable = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("**").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + + Table table = + DeIdentifyTableConditionMasking.deIdentifyTableConditionMasking( + PROJECT_ID, tableToDeIdentify); + + String output = bout.toString(); + assertThat(output).contains("Table after de-identification:"); + assertThat(table).isEqualTo(expectedTable); + } + + @Test + public void testDeIdentifyTableInfoTypes() throws IOException { + // Transform findings found in column + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "Charles Dickens name was a curse invented by Shakespeare.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in Jane Austen's novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) + .build()) + .build(); + Table expectedTable = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "[PERSON_NAME] name was a curse invented by [PERSON_NAME].") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in [PERSON_NAME] novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues( + Value.newBuilder().setStringValue("[PERSON_NAME] loved cats.").build()) + .build()) + .build(); + + Table table = DeIdentifyTableInfoTypes.deIdentifyTableInfoTypes(PROJECT_ID, tableToDeIdentify); + + String output = bout.toString(); + assertThat(output).contains("Table after de-identification:"); + assertThat(table).isEqualTo(expectedTable); + } + + @Test + public void testDeIdentifyTableRowSuppress() throws IOException { + // Suppress a row based on the content of a column + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + Table expectedTable = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .build()) + .build(); + + Table table = + DeIdentifyTableRowSuppress.deIdentifyTableRowSuppress(PROJECT_ID, tableToDeIdentify); + + String output = bout.toString(); + assertThat(output).contains("Table after de-identification:"); + assertThat(table).isEqualTo(expectedTable); + } + + @Test + public void testDeIdentifyTableConditionsInfoTypes() throws IOException { + // Transform findings only when specific conditions are met on another field + Table tableToDeIdentify = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "Charles Dickens name was a curse invented by Shakespeare.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in Jane Austen's novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) + .build()) + .build(); + Table expectedTable = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("AGE").build()) + .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) + .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) + .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("101").build()) + .addValues(Value.newBuilder().setStringValue("[PERSON_NAME]").build()) + .addValues(Value.newBuilder().setStringValue("95").build()) + .addValues( + Value.newBuilder() + .setStringValue( + "[PERSON_NAME] name was a curse invented by [PERSON_NAME].") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("22").build()) + .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) + .addValues(Value.newBuilder().setStringValue("21").build()) + .addValues( + Value.newBuilder() + .setStringValue("There are 14 kisses in Jane Austen's novels.") + .build()) + .build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("55").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) + .addValues(Value.newBuilder().setStringValue("75").build()) + .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) + .build()) + .build(); + + Table table = + DeIdentifyTableConditionInfoTypes.deIdentifyTableConditionInfoTypes( + PROJECT_ID, tableToDeIdentify); + + String output = bout.toString(); + assertThat(output).contains("Table after de-identification:"); + assertThat(table).isEqualTo(expectedTable); + } + + @Test + public void testDeIdentifyWithDateShift() throws IOException { + Path inputFile = Paths.get("src/test/resources/dates.csv"); + assertWithMessage("Input file must exist").that(inputFile.toFile().exists()).isTrue(); + Path outputFile = Paths.get("src/test/resources/results.csv"); + assertWithMessage("Output file must be writeable").that(inputFile.toFile().canWrite()).isTrue(); + DeIdentifyWithDateShift.deIdentifyWithDateShift(PROJECT_ID, inputFile, outputFile); + + String output = bout.toString(); + assertThat(output).contains("Content written to file: "); + + // Clean up test output + Files.delete(outputFile); + } + + @Test + public void testDeIdentifyWithRedaction() throws IOException { + DeIdentifyWithRedaction.deIdentifyWithRedaction( + PROJECT_ID, "My name is Alicia Abernathy, and my email address is aabernathy@example.com."); + + String output = bout.toString(); + assertThat(output) + .contains( + "Text after redaction: " + "My name is Alicia Abernathy, and my email address is ."); + } + + @Test + public void testDeIdentifyWithReplacement() throws IOException { + DeIdentifyWithReplacement.deIdentifyWithReplacement( + PROJECT_ID, "My name is Alicia Abernathy, and my email address is aabernathy@example.com."); + + String output = bout.toString(); + assertThat(output) + .contains( + "Text after redaction: " + + "My name is Alicia Abernathy, and my email address is [email-address]."); + } + + @Test + public void testDeIdentifyWithInfoType() throws IOException { + DeIdentifyWithInfoType.deIdentifyWithInfoType(PROJECT_ID, "My email is test@example.com"); + + String output = bout.toString(); + assertThat(output).contains("Text after redaction: " + "My email is [EMAIL_ADDRESS]"); + } + + @Test + public void testDeIdentifyWithSimpleWordList() throws IOException { + DeIdentifyWithSimpleWordList.deidentifyWithSimpleWordList( + PROJECT_ID, "Patient was seen in RM-YELLOW then transferred to rm green."); + + String output = bout.toString(); + assertThat(output).contains("Text after replace with infotype config: "); + } + + @Test + public void testDeIdentifyWithExceptionList() throws IOException { + DeIdentifyWithExceptionList.deIdentifyWithExceptionList( + PROJECT_ID, "jack@example.org accessed customer record of user5@example.com"); + + String output = bout.toString(); + assertThat(output).contains("Text after replace with infotype config: "); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/InfoTypesTests.java b/dlp/snippets/src/test/java/dlp/snippets/InfoTypesTests.java new file mode 100644 index 00000000000..39792218457 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/InfoTypesTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class InfoTypesTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS"); + } + + @Test + public void testListInfoTypes() throws Exception { + InfoTypesList.listInfoTypes(); + String output = bout.toString(); + assertThat(output).contains("Name"); + assertThat(output).contains("Display name"); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/InspectTests.java b/dlp/snippets/src/test/java/dlp/snippets/InspectTests.java new file mode 100644 index 00000000000..7ed5f4787be --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/InspectTests.java @@ -0,0 +1,410 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.common.collect.ImmutableList; +import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.Table; +import com.google.privacy.dlp.v2.Table.Row; +import com.google.privacy.dlp.v2.Value; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.TopicName; +import java.util.Arrays; +import java.util.UUID; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class InspectTests extends TestBase { + + // TODO: Update as ENV_VARs + private static final String datastoreNamespace = ""; + private static final String datastoreKind = "dlp"; + private static final String DOCUMENT_INPUT_FILE = "src/test/resources/sensitive-data-image.jpg"; + + private static final UUID testRunUuid = UUID.randomUUID(); + private static final TopicName topicName = + TopicName.of(PROJECT_ID, String.format("%s-%s", TOPIC_ID, testRunUuid)); + private static final ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of( + PROJECT_ID, String.format("%s-%s", SUBSCRIPTION_ID, testRunUuid.toString())); + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of( + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GCS_PATH", + "PUB_SUB_TOPIC", + "PUB_SUB_SUBSCRIPTION", + "BIGQUERY_DATASET", + "BIGQUERY_TABLE"); + } + + @BeforeClass + public static void before() throws Exception { + // Create a new topic + try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) { + topicAdminClient.createTopic(topicName); + } + + // Create a new subscription + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) { + subscriptionAdminClient.createSubscription( + subscriptionName, topicName, PushConfig.getDefaultInstance(), 0); + } + } + + @AfterClass + public static void after() throws Exception { + // Delete the test topic + try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) { + topicAdminClient.deleteTopic(topicName); + } catch (ApiException e) { + System.err.println(String.format("Error deleting topic %s: %s", topicName.getTopic(), e)); + // Keep trying to clean up + } + + // Delete the test subscription + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) { + subscriptionAdminClient.deleteSubscription(subscriptionName); + } catch (ApiException e) { + System.err.println( + String.format( + "Error deleting subscription %s: %s", subscriptionName.getSubscription(), e)); + // Keep trying to clean up + } + } + + + @Test + public void testInspectPhoneNumber() throws Exception { + InspectString.inspectString(PROJECT_ID, "My phone number is (415) 555-0890"); + + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + } + + @Test + public void testInspectString() throws Exception { + InspectString.inspectString(PROJECT_ID, "I'm Gary and my email is gary@example.com"); + + String output = bout.toString(); + assertThat(output).contains("Info type: EMAIL_ADDRESS"); + } + + @Test + public void testInspectWithCustomRegex() throws Exception { + InspectWithCustomRegex.inspectWithCustomRegex( + PROJECT_ID, "Patients MRN 444-5-22222", "[1-9]{3}-[1-9]{1}-[1-9]{5}"); + + String output = bout.toString(); + assertThat(output).contains("Info type: C_MRN"); + } + + @Test + public void testInspectStringWithExclusionDict() throws Exception { + InspectStringWithExclusionDict.inspectStringWithExclusionDict( + PROJECT_ID, + "Some email addresses: gary@example.com, example@example.com", + Arrays.asList("example@example.com")); + + String output = bout.toString(); + assertThat(output).contains("gary@example.com"); + assertThat(output).doesNotContain("example@example.com"); + } + + @Test + public void testInspectStringWithExclusionDictSubstring() throws Exception { + InspectStringWithExclusionDictSubstring.inspectStringWithExclusionDictSubstring( + PROJECT_ID, + "Some email addresses: gary@example.com, TEST@example.com", + Arrays.asList("TEST")); + + String output = bout.toString(); + assertThat(output).contains("gary@example.com"); + assertThat(output).doesNotContain("TEST@example.com"); + } + + @Test + public void testInspectStringWithExclusionRegex() throws Exception { + InspectStringWithExclusionRegex.inspectStringWithExclusionRegex( + PROJECT_ID, "Some email addresses: gary@example.com, bob@example.org", ".+@example.com"); + + String output = bout.toString(); + assertThat(output).contains("bob@example.org"); + assertThat(output).doesNotContain("gary@example.com"); + } + + @Test + public void testInspectStringCustomExcludingSubstring() throws Exception { + InspectStringCustomExcludingSubstring.inspectStringCustomExcludingSubstring( + PROJECT_ID, + "Name: Doe, John. Name: Example, Jimmy", + "[A-Z][a-z]{1,15}, [A-Z][a-z]{1,15}", + Arrays.asList("Jimmy")); + + String output = bout.toString(); + assertThat(output).contains("Doe, John"); + assertThat(output).doesNotContain("Example, Jimmy"); + } + + @Test + public void testInspectStringCustomOmitOverlap() throws Exception { + InspectStringCustomOmitOverlap.inspectStringCustomOmitOverlap( + PROJECT_ID, "Name: Jane Doe. Name: Larry Page."); + + String output = bout.toString(); + assertThat(output).contains("Jane Doe"); + assertThat(output).doesNotContain("Larry Page"); + } + + @Test + public void testInspectStringOmitOverlap() throws Exception { + InspectStringOmitOverlap.inspectStringOmitOverlap(PROJECT_ID, "james@example.com"); + + String output = bout.toString(); + assertThat(output).contains("EMAIL_ADDRESS"); + assertThat(output).doesNotContain("PERSON_NAME"); + } + + @Test + public void testInspectStringWithoutOverlap() throws Exception { + InspectStringWithoutOverlap.inspectStringWithoutOverlap( + PROJECT_ID, "example.com is a domain, james@example.org is an email."); + + String output = bout.toString(); + assertThat(output).contains("example.com"); + assertThat(output).doesNotContain("example.org"); + } + + @Test + public void testInspectTable() { + Table tableToInspect = + Table.newBuilder() + .addHeaders(FieldId.newBuilder().setName("name").build()) + .addHeaders(FieldId.newBuilder().setName("phone").build()) + .addRows( + Row.newBuilder() + .addValues(Value.newBuilder().setStringValue("John Doe").build()) + .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build())) + .build(); + InspectTable.inspectTable(PROJECT_ID, tableToInspect); + + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + } + + @Test + public void testInspectStringCustomHotword() throws Exception { + InspectStringCustomHotword.inspectStringCustomHotword( + PROJECT_ID, "patient name: John Doe", "patient"); + + String output = bout.toString(); + assertThat(output).contains("John Doe"); + } + + @Test + public void testInspectStringCustomHotwordNegativeExample() throws Exception { + InspectStringCustomHotword.inspectStringCustomHotword(PROJECT_ID, "name: John Doe", "patient"); + + String output = bout.toString(); + assertThat(output).doesNotContain("John Doe"); + } + + @Test + public void testInspectStringMultipleRulesPatientRule() throws Exception { + InspectStringMultipleRules.inspectStringMultipleRules(PROJECT_ID, "patient name: Jane Doe"); + + String output = bout.toString(); + assertThat(output).contains("LIKELY"); + } + + @Test + public void testInspectStringMultipleRulesDoctorRule() throws Exception { + InspectStringMultipleRules.inspectStringMultipleRules(PROJECT_ID, "doctor: Jane Doe"); + + String output = bout.toString(); + assertThat(output).contains("Findings: 0"); + } + + @Test + public void testInspectStringMultipleRulesQuasimodoRule() throws Exception { + InspectStringMultipleRules.inspectStringMultipleRules(PROJECT_ID, "patient: Quasimodo"); + + String output = bout.toString(); + assertThat(output).contains("Findings: 0"); + } + + @Test + public void testInspectStringMultipleRulesRedactedRule() throws Exception { + InspectStringMultipleRules.inspectStringMultipleRules(PROJECT_ID, "name of patient: REDACTED"); + + String output = bout.toString(); + assertThat(output).contains("Findings: 0"); + } + + @Test + public void textInspectTestFile() throws Exception { + InspectTextFile.inspectTextFile(PROJECT_ID, "src/test/resources/test.txt"); + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + assertThat(output).contains("Info type: EMAIL_ADDRESS"); + } + + @Test + public void testInspectImageFile() throws Exception { + InspectImageFile.inspectImageFile(PROJECT_ID, "src/test/resources/test.png"); + + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + assertThat(output).contains("Info type: EMAIL_ADDRESS"); + } + + @Test + public void testRedactImageAllInfoTypes() throws Exception { + InspectImageFileAllInfoTypes.inspectImageFileAllInfoTypes(PROJECT_ID, DOCUMENT_INPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + assertThat(output).contains("Info type: EMAIL_ADDRESS"); + assertThat(output).contains("Info type: DATE"); + } + + @Test + public void testRedactImageListedInfoTypes() throws Exception { + InspectImageFileListedInfoTypes.inspectImageFileListedInfoTypes( + PROJECT_ID, DOCUMENT_INPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Info type: PHONE_NUMBER"); + assertThat(output).contains("Info type: EMAIL_ADDRESS"); + assertThat(output).doesNotContain("Info type: DATE"); + } + + @Test + public void testInspectGcsFile() throws Exception { + InspectGcsFile.inspectGcsFile( + PROJECT_ID, GCS_PATH, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + assertThat(output).contains("Job status: DONE"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testInspectGcsFileWithSampling() throws Exception { + InspectGcsFileWithSampling.inspectGcsFileWithSampling( + PROJECT_ID, GCS_PATH, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + assertThat(output).contains("Job status: DONE"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testInspectDatastoreEntity() throws Exception { + InspectDatastoreEntity.insepctDatastoreEntity( + PROJECT_ID, + datastoreNamespace, + datastoreKind, + topicName.getTopic(), + subscriptionName.getSubscription()); + + String output = bout.toString(); + assertThat(output).contains("Job status: DONE"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testInspectBigQueryTable() throws Exception { + InspectBigQueryTable.inspectBigQueryTable( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + assertThat(output).contains("Job status: DONE"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testInspectBigQueryTableWithSampling() throws Exception { + InspectBigQueryTableWithSampling.inspectBigQueryTableWithSampling( + PROJECT_ID, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + assertThat(output).contains("Job status: DONE"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testInspectWithHotwordRules() throws Exception { + InspectWithHotwordRules.inspectWithHotwordRules( + PROJECT_ID, + "Patient's MRN 444-5-22222 and just a number 333-2-33333", + "[1-9]{3}-[1-9]{1}-[1-9]{5}", + "(?i)(mrn|medical)(?-i)"); + + String output = bout.toString(); + assertThat(output).contains("Findings: 2"); + assertThat(output).contains("Info type: C_MRN"); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/JobsTests.java b/dlp/snippets/src/test/java/dlp/snippets/JobsTests.java new file mode 100644 index 00000000000..7a0a13551fb --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/JobsTests.java @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.collect.ImmutableList; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CloudStorageOptions.FileSet; +import com.google.privacy.dlp.v2.CreateDlpJobRequest; +import com.google.privacy.dlp.v2.DeleteDlpJobRequest; +import com.google.privacy.dlp.v2.DlpJob; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.StorageConfig; +import java.io.IOException; +import java.util.UUID; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class JobsTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "GCS_PATH"); + } + + private static DlpJob createJob(String jobId) throws IOException { + try (DlpServiceClient dlp = DlpServiceClient.create()) { + FileSet fileSet = FileSet.newBuilder().setUrl(GCS_PATH).build(); + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder().setFileSet(fileSet).build(); + StorageConfig storageConfig = + StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build(); + + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setStorageConfig(storageConfig) + .setInspectConfig(InspectConfig.newBuilder().build()) + .build(); + + CreateDlpJobRequest createDlpJobRequest = + CreateDlpJobRequest.newBuilder() + .setParent(LocationName.of(PROJECT_ID, "global").toString()) + .setInspectJob(inspectJobConfig) + .setJobId(jobId) + .build(); + + return dlp.createDlpJob(createDlpJobRequest); + } + } + + @Test + public void testCreateJobs() throws Exception { + // Call createJobs to create a Dlp job from project id and gcs path. + JobsCreate.createJobs(PROJECT_ID, GCS_PATH); + String output = bout.toString(); + assertThat(output).contains("Job created successfully:"); + + // Delete the created Dlp Job + String dlpJobName = output.split("Job created successfully: ")[1].split("\n")[0]; + DeleteDlpJobRequest deleteDlpJobRequest = + DeleteDlpJobRequest.newBuilder().setName(dlpJobName).build(); + try (DlpServiceClient client = DlpServiceClient.create()) { + client.deleteDlpJob(deleteDlpJobRequest); + } + } + + @Test + public void testGetJobs() throws Exception { + // Create a job with a unique UUID to be gotten + String jobId = UUID.randomUUID().toString(); + DlpJob createdDlpJob = createJob(jobId); + + // Get the job with the specified ID + JobsGet.getJobs(PROJECT_ID, "i-" + jobId); + String output = bout.toString(); + assertThat(output).contains("Job got successfully."); + + // Delete the created Dlp Job + String dlpJobName = createdDlpJob.getName(); + DeleteDlpJobRequest deleteDlpJobRequest = + DeleteDlpJobRequest.newBuilder().setName(dlpJobName).build(); + try (DlpServiceClient client = DlpServiceClient.create()) { + client.deleteDlpJob(deleteDlpJobRequest); + } + } + + @Test + public void testListJobs() throws Exception { + // Call listJobs to print out a list of jobIds + JobsList.listJobs(PROJECT_ID); + String output = bout.toString(); + + // Check that the output contains a list of jobs, or is empty + assertThat(output).contains("DLP jobs found:"); + } + + @Test + public void testDeleteJobs() throws Exception { + // Create a job with a unique UUID to be deleted + String jobId = UUID.randomUUID().toString(); + createJob(jobId); + + // Delete the job with the specified ID + JobsDelete.deleteJobs(PROJECT_ID, "i-" + jobId); + String output = bout.toString(); + assertThat(output).contains("Job deleted successfully."); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/QuickstartTests.java b/dlp/snippets/src/test/java/dlp/snippets/QuickstartTests.java new file mode 100644 index 00000000000..eb333a4ac3a --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/QuickstartTests.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class QuickstartTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT"); + } + + @Test + public void testQuickstart() throws IOException { + QuickStart.quickstart(PROJECT_ID); + + String output = bout.toString(); + assertThat(output).contains("Inspect of text complete"); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/RedactTests.java b/dlp/snippets/src/test/java/dlp/snippets/RedactTests.java new file mode 100644 index 00000000000..13c0386e340 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/RedactTests.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class RedactTests extends TestBase { + + private static final String SIMPLE_INPUT_FILE = "src/test/resources/test.png"; + private static final String SIMPLE_OUTPUT_FILE = "redacted.png"; + private static final String DOCUMENT_INPUT_FILE = "src/test/resources/sensitive-data-image.jpg"; + private static final String DOCUMENT_OUTPUT_FILE = "sensitive-data-image-redacted.jpg"; + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT"); + } + + @After + public void after() throws IOException { + Files.deleteIfExists(Paths.get(SIMPLE_OUTPUT_FILE)); + Files.deleteIfExists(Paths.get(DOCUMENT_OUTPUT_FILE)); + } + + @Test + public void testRedactImage() throws Exception { + RedactImageFile.redactImageFile(PROJECT_ID, SIMPLE_INPUT_FILE, SIMPLE_OUTPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Redacted image written"); + } + + @Test + public void testRedactImageAllInfoTypes() throws Exception { + RedactImageFileAllInfoTypes.redactImageFileAllInfoTypes( + PROJECT_ID, DOCUMENT_INPUT_FILE, DOCUMENT_OUTPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Redacted image written"); + } + + @Test + public void testRedactImageListedInfoTypes() throws Exception { + RedactImageFileListedInfoTypes.redactImageFileListedInfoTypes( + PROJECT_ID, DOCUMENT_INPUT_FILE, DOCUMENT_OUTPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Redacted image written"); + } + + @Test + public void testRedactImageColoredInfoTypes() throws Exception { + RedactImageFileColoredInfoTypes.redactImageFileColoredInfoTypes( + PROJECT_ID, DOCUMENT_INPUT_FILE, DOCUMENT_OUTPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Redacted image written"); + } + + @Test + public void testRedactImageAllText() throws Exception { + RedactImageFileAllText.redactImageFileAllText( + PROJECT_ID, DOCUMENT_INPUT_FILE, DOCUMENT_OUTPUT_FILE); + + String output = bout.toString(); + assertThat(output).contains("Redacted image written"); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/RiskAnalysisTests.java b/dlp/snippets/src/test/java/dlp/snippets/RiskAnalysisTests.java new file mode 100644 index 00000000000..034ee2d9808 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/RiskAnalysisTests.java @@ -0,0 +1,183 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.common.collect.ImmutableList; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.TopicName; +import java.util.Arrays; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") +@RunWith(JUnit4.class) +public class RiskAnalysisTests extends TestBase { + + private UUID testRunUuid = UUID.randomUUID(); + private TopicName topicName = + TopicName.of(PROJECT_ID, String.format("%s-%s", TOPIC_ID, testRunUuid.toString())); + private ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of( + PROJECT_ID, String.format("%s-%s", SUBSCRIPTION_ID, testRunUuid.toString())); + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of( + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "PUB_SUB_TOPIC", + "PUB_SUB_SUBSCRIPTION", + "BIGQUERY_DATASET", + "BIGQUERY_TABLE"); + } + + @Before + public void before() throws Exception { + // Create a new topic + try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) { + topicAdminClient.createTopic(topicName); + } + // Create a new subscription + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) { + subscriptionAdminClient.createSubscription( + subscriptionName, topicName, PushConfig.getDefaultInstance(), 0); + } + } + + @After + public void after() throws Exception { + // Delete the test topic + try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) { + topicAdminClient.deleteTopic(topicName); + } catch (ApiException e) { + System.err.println(String.format("Error deleting topic %s: %s", topicName.getTopic(), e)); + // Keep trying to clean up + } + + // Delete the test subscription + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) { + subscriptionAdminClient.deleteSubscription(subscriptionName); + } catch (ApiException e) { + System.err.println( + String.format( + "Error deleting subscription %s: %s", subscriptionName.getSubscription(), e)); + // Keep trying to clean up + } + } + + @Test + public void testNumericalStats() throws Exception { + RiskAnalysisNumericalStats.numericalStatsAnalysis( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + String output = bout.toString(); + assertThat(output).contains("Value at "); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testCategoricalStats() throws Exception { + RiskAnalysisCategoricalStats.categoricalStatsAnalysis( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + + assertThat(output).containsMatch("Most common value occurs \\d time"); + assertThat(output).containsMatch("Least common value occurs \\d time"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testKAnonymity() throws Exception { + RiskAnalysisKAnonymity.calculateKAnonymity( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + String output = bout.toString(); + assertThat(output).containsMatch("Bucket size range: \\[\\d, \\d\\]"); + assertThat(output).contains("Quasi-ID values: integer_value: 19"); + assertThat(output).contains("Class size: 1"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testLDiversity() throws Exception { + RiskAnalysisLDiversity.calculateLDiversity( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + String output = bout.toString(); + assertThat(output).contains("Quasi-ID values: integer_value: 19"); + assertThat(output).contains("Class size: 1"); + assertThat(output).contains("Sensitive value string_value: \"James\""); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } + + @Test + public void testKMap() throws Exception { + RiskAnalysisKMap.calculateKMap( + PROJECT_ID, DATASET_ID, TABLE_ID, topicName.getTopic(), subscriptionName.getSubscription()); + + String output = bout.toString(); + + assertThat(output).containsMatch("Anonymity range: \\[\\d, \\d]"); + assertThat(output).containsMatch("Size: \\d"); + assertThat(output).containsMatch("Values: \\{\\d{2}, \"Female\"\\}"); + String jobName = Arrays.stream(output.split("\n")) + .filter(line -> line.contains("Job name:")) + .findFirst() + .get(); + jobName = jobName.split(":")[1].trim(); + try (DlpServiceClient dlp = DlpServiceClient.create()) { + dlp.deleteDlpJob(jobName); + } + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/TemplatesTests.java b/dlp/snippets/src/test/java/dlp/snippets/TemplatesTests.java new file mode 100644 index 00000000000..1af435a56c4 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/TemplatesTests.java @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.collect.ImmutableList; +import com.google.privacy.dlp.v2.CreateInspectTemplateRequest; +import com.google.privacy.dlp.v2.DeleteInspectTemplateRequest; +import com.google.privacy.dlp.v2.InfoType; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectTemplate; +import com.google.privacy.dlp.v2.LocationName; +import java.io.IOException; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class TemplatesTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT"); + } + + private static InspectTemplate createTemplate() throws IOException { + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + List infoTypes = + Stream.of("PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD_NUMBER") + .map(it -> InfoType.newBuilder().setName(it).build()) + .collect(Collectors.toList()); + + InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).build(); + + InspectTemplate inspectTemplate = + InspectTemplate.newBuilder().setInspectConfig(inspectConfig).build(); + + CreateInspectTemplateRequest createInspectTemplateRequest = + CreateInspectTemplateRequest.newBuilder() + .setParent(LocationName.of(PROJECT_ID, "global").toString()) + .setInspectTemplate(inspectTemplate) + .build(); + + return dlpServiceClient.createInspectTemplate(createInspectTemplateRequest); + } + } + + @Test + public void testCreateInspectTemplate() throws Exception { + TemplatesCreate.createInspectTemplate(PROJECT_ID); + String output = bout.toString(); + assertThat(output).contains("Template created: "); + + // Delete the created template + String templateId = output.split("Template created: ")[1].split("\n")[0]; + DeleteInspectTemplateRequest deleteInspectTemplateRequest = + DeleteInspectTemplateRequest.newBuilder().setName(templateId).build(); + try (DlpServiceClient client = DlpServiceClient.create()) { + client.deleteInspectTemplate(deleteInspectTemplateRequest); + } + } + + @Test + public void testListInspectTemplate() throws Exception { + TemplatesList.listInspectTemplates(PROJECT_ID); + String output = bout.toString(); + assertThat(output).contains("Templates found:"); + } + + @Test + public void testDeleteInspectTemplate() throws Exception { + // Create a template to be deleted and extract its ID + InspectTemplate template = createTemplate(); + String templateName = template.getName(); + String templateId; + + Matcher matcher = Pattern.compile("inspectTemplates/").matcher(templateName); + if (matcher.find()) { + templateId = templateName.substring(matcher.end()); + } else { + throw new Exception("Could not extract templateId"); + } + + // Delete the template with the specified ID + TemplatesDelete.deleteInspectTemplate(PROJECT_ID, templateId); + String output = bout.toString(); + assertThat(output).contains("Deleted template:"); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/TestBase.java b/dlp/snippets/src/test/java/dlp/snippets/TestBase.java new file mode 100644 index 00000000000..58ed908bb1e --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/TestBase.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.collect.ImmutableList; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; + +/** Common base class for DLP snippet tests */ +abstract class TestBase { + + protected static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + protected static final String GCS_PATH = System.getenv("GCS_PATH"); + protected static final String TOPIC_ID = System.getenv("PUB_SUB_TOPIC"); + protected static final String SUBSCRIPTION_ID = System.getenv("PUB_SUB_SUBSCRIPTION"); + protected static final String DATASET_ID = System.getenv("BIGQUERY_DATASET"); + protected static final String TABLE_ID = System.getenv("BIGQUERY_TABLE"); + protected static final String wrappedKey = System.getenv("DLP_DEID_WRAPPED_KEY"); + protected static final String kmsKeyName = System.getenv("DLP_DEID_KEY_NAME"); + + protected ByteArrayOutputStream bout; + private PrintStream originalOut = System.out; + + protected abstract ImmutableList requiredEnvVars(); + + private static void requireEnvVar(String varName) { + assertWithMessage( + String.format("Environment variable '%s' must be set to perform these tests.", varName)) + .that(System.getenv(varName)) + .isNotEmpty(); + } + + @Before + public void beforeBase() { + requiredEnvVars().stream().forEach(TestBase::requireEnvVar); + + // Capture stdout + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @After + public void afterBase() { + // Restore stdout + System.setOut(originalOut); + bout.reset(); + } +} diff --git a/dlp/snippets/src/test/java/dlp/snippets/TriggersTests.java b/dlp/snippets/src/test/java/dlp/snippets/TriggersTests.java new file mode 100644 index 00000000000..b84710fb345 --- /dev/null +++ b/dlp/snippets/src/test/java/dlp/snippets/TriggersTests.java @@ -0,0 +1,123 @@ +/* + * Copyright 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp.snippets; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.common.collect.ImmutableList; +import com.google.privacy.dlp.v2.CloudStorageOptions; +import com.google.privacy.dlp.v2.CreateJobTriggerRequest; +import com.google.privacy.dlp.v2.DeleteJobTriggerRequest; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectJobConfig; +import com.google.privacy.dlp.v2.JobTrigger; +import com.google.privacy.dlp.v2.LocationName; +import com.google.privacy.dlp.v2.Schedule; +import com.google.privacy.dlp.v2.StorageConfig; +import com.google.protobuf.Duration; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class TriggersTests extends TestBase { + + @Override + protected ImmutableList requiredEnvVars() { + return ImmutableList.of("GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "GCS_PATH"); + } + + private static JobTrigger createTrigger() throws IOException { + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder() + .setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl(GCS_PATH)) + .build(); + StorageConfig storageConfig = + StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build(); + + InspectJobConfig inspectJobConfig = + InspectJobConfig.newBuilder() + .setInspectConfig(InspectConfig.newBuilder().build()) + .setStorageConfig(storageConfig) + .build(); + + Duration duration = Duration.newBuilder().setSeconds(24 * 3600).build(); + Schedule schedule = Schedule.newBuilder().setRecurrencePeriodDuration(duration).build(); + JobTrigger.Trigger trigger = JobTrigger.Trigger.newBuilder().setSchedule(schedule).build(); + JobTrigger jobTrigger = + JobTrigger.newBuilder() + .setInspectJob(inspectJobConfig) + .setStatus(JobTrigger.Status.HEALTHY) + .addTriggers(trigger) + .build(); + + CreateJobTriggerRequest createJobTriggerRequest = + CreateJobTriggerRequest.newBuilder() + .setParent(LocationName.of(PROJECT_ID, "global").toString()) + .setJobTrigger(jobTrigger) + .build(); + + return dlpServiceClient.createJobTrigger(createJobTriggerRequest); + } + } + + @Test + public void testCreateTrigger() throws Exception { + TriggersCreate.createTrigger(PROJECT_ID, GCS_PATH); + String output = bout.toString(); + assertThat(output).contains("Created Trigger:"); + + // Delete the created trigger + String triggerId = output.split("Created Trigger: ")[1].split("\n")[0]; + DeleteJobTriggerRequest deleteJobTriggerRequest = + DeleteJobTriggerRequest.newBuilder().setName(triggerId).build(); + try (DlpServiceClient client = DlpServiceClient.create()) { + client.deleteJobTrigger(deleteJobTriggerRequest); + } + } + + @Test + public void testListTrigger() throws Exception { + TriggersList.listTriggers(PROJECT_ID); + String output = bout.toString(); + assertThat(output).contains("DLP triggers found:"); + } + + @Test + public void testDeleteTrigger() throws Exception { + JobTrigger trigger = createTrigger(); + String triggerName = trigger.getName(); + String triggerId; + + Matcher matcher = Pattern.compile("jobTriggers/").matcher(triggerName); + if (matcher.find()) { + triggerId = triggerName.substring(matcher.end()); + } else { + throw new Exception("Could not extract triggerID"); + } + + // Delete the job with the specified ID + TriggersDelete.deleteTrigger(PROJECT_ID, triggerId); + String output = bout.toString(); + assertThat(output).contains("Trigger deleted:"); + } +} diff --git a/dlp/snippets/src/test/resources/dates.csv b/dlp/snippets/src/test/resources/dates.csv new file mode 100644 index 00000000000..290a85dec68 --- /dev/null +++ b/dlp/snippets/src/test/resources/dates.csv @@ -0,0 +1,5 @@ +name,birth_date,credit_card,register_date +Ann,01/01/1970,4532908762519852,07/21/1996 +James,03/06/1988,4301261899725540,04/09/2001 +Dan,08/14/1945,4620761856015295,11/15/2011 +Laura,11/03/1992,4564981067258901,01/04/2017 diff --git a/dlp/snippets/src/test/resources/results.correct.csv b/dlp/snippets/src/test/resources/results.correct.csv new file mode 100644 index 00000000000..5b078fe825a --- /dev/null +++ b/dlp/snippets/src/test/resources/results.correct.csv @@ -0,0 +1,5 @@ +name,birth_date,credit_card,register_date +Ann,1970-01-06,4532908762519852,1996-07-26 +James,1988-03-11,4301261899725540,2001-04-14 +Dan,1945-08-19,4620761856015295,2011-11-20 +Laura,1992-11-08,4564981067258901,2017-01-09 diff --git a/dlp/snippets/src/test/resources/sensitive-data-image.jpg b/dlp/snippets/src/test/resources/sensitive-data-image.jpg new file mode 100644 index 00000000000..6e2d84546e4 Binary files /dev/null and b/dlp/snippets/src/test/resources/sensitive-data-image.jpg differ diff --git a/dlp/snippets/src/test/resources/test.png b/dlp/snippets/src/test/resources/test.png new file mode 100644 index 00000000000..748f46cdcb5 Binary files /dev/null and b/dlp/snippets/src/test/resources/test.png differ diff --git a/dlp/snippets/src/test/resources/test.txt b/dlp/snippets/src/test/resources/test.txt new file mode 100644 index 00000000000..f30af240c72 --- /dev/null +++ b/dlp/snippets/src/test/resources/test.txt @@ -0,0 +1 @@ +My phone number is (223) 456-7890 and my email address is gary@example.com. \ No newline at end of file