Merge branch 'main' into renovate/c8-8.x

cloud-tasks/tutorial-gcf/app/createTask.js

@@ -45,7 +45,7 @@ const createHttpTaskWithToken = async function (
       url,
       oidcToken: {
         serviceAccountEmail: email,
-        audience: new URL(url).origin,
+        audience: url,
       },
       headers: {
         'Content-Type': 'application/json',

dlp/createStoredInfoType.js

@@ -0,0 +1,111 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+// sample-metadata:
+// title: Create stored infotype.
+// description: Uses the Data Loss Prevention API to create a stored infotype.
+// usage: node createStoredInfoType.js projectId  infoTypeId, outputPath, dataProjectId, datasetId, tableId, fieldName
+function main(
+  projectId,
+  infoTypeId,
+  outputPath,
+  dataProjectId,
+  datasetId,
+  tableId,
+  fieldName
+) {
+  // [START dlp_create_stored_infotype]
+  // Import the required libraries
+  const dlp = require('@google-cloud/dlp');
+
+  // Create a DLP client
+  const dlpClient = new dlp.DlpServiceClient();
+
+  // The project ID to run the API call under.
+  // const projectId = "your-project-id";
+
+  // The identifier for the stored infoType
+  // const infoTypeId = 'github-usernames';
+
+  // The path to the location in a Cloud Storage bucket to store the created dictionary
+  // const outputPath = 'cloud-bucket-path';
+
+  // The project ID the table is stored under
+  // This may or (for public datasets) may not equal the calling project ID
+  // const dataProjectId = 'my-project';
+
+  // The ID of the dataset to inspect, e.g. 'my_dataset'
+  // const datasetId = 'my_dataset';
+
+  // The ID of the table to inspect, e.g. 'my_table'
+  // const tableId = 'my_table';
+
+  // Field ID to be used for constructing dictionary
+  // const fieldName = 'field_name';
+
+  async function createStoredInfoType() {
+    // The name you want to give the dictionary.
+    const displayName = 'GitHub usernames';
+    // A description of the dictionary.
+    const description = 'Dictionary of GitHub usernames used in commits';
+
+    // Specify configuration for the large custom dictionary
+    const largeCustomDictionaryConfig = {
+      outputPath: {
+        path: outputPath,
+      },
+      bigQueryField: {
+        table: {
+          datasetId: datasetId,
+          projectId: dataProjectId,
+          tableId: tableId,
+        },
+        field: {
+          name: fieldName,
+        },
+      },
+    };
+
+    // Stored infoType configuration that uses large custom dictionary.
+    const storedInfoTypeConfig = {
+      displayName: displayName,
+      description: description,
+      largeCustomDictionary: largeCustomDictionaryConfig,
+    };
+
+    // Construct the job creation request to be sent by the client.
+    const request = {
+      parent: `projects/${projectId}/locations/global`,
+      config: storedInfoTypeConfig,
+      storedInfoTypeId: infoTypeId,
+    };
+
+    // Send the job creation request and process the response.
+    const [response] = await dlpClient.createStoredInfoType(request);
+
+    // Print results
+    console.log(`InfoType stored successfully: ${response.name}`);
+  }
+  createStoredInfoType();
+  // [END dlp_create_stored_infotype]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+
+main(...process.argv.slice(2));

dlp/deidentifyWithTimeExtraction.js

@@ -0,0 +1,110 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+// sample-metadata:
+//  title: De-identify with time extraction
+//  description: De-identify sensitive data in a string by replacing it with a given time config.
+//  usage: node deidentifyWithTimeExtraction.js my-project string
+function main(projectId) {
+  // [START dlp_deidentify_time_extract]
+  // Imports the Google Cloud Data Loss Prevention library
+  const DLP = require('@google-cloud/dlp');
+
+  // Instantiates a client
+  const dlp = new DLP.DlpServiceClient();
+
+  // The project ID to run the API call under
+  // const projectId = 'my-project';
+
+  // The string to de-identify
+  // const string = 'My BirthDay is on 9/21/1976';
+
+  // Table to de-identify
+  const tablularData = {
+    headers: [
+      {name: 'Name'},
+      {name: 'Birth Date'},
+      {name: 'Credit Card'},
+      {name: 'Register Date'},
+    ],
+    rows: [
+      {
+        values: [
+          {stringValue: 'Ann'},
+          {stringValue: '01/01/1970'},
+          {stringValue: '4532908762519852'},
+          {stringValue: '07/21/1996'},
+        ],
+      },
+      {
+        values: [
+          {stringValue: 'James'},
+          {stringValue: '03/06/1988'},
+          {stringValue: '4301261899725540'},
+          {stringValue: '04/09/2001'},
+        ],
+      },
+    ],
+  };
+
+  async function deidentifyWithTimeExtraction() {
+    // Specify transformation to extract a portion of date.
+    const primitiveTransformation = {
+      timePartConfig: {
+        partToExtract: 'YEAR',
+      },
+    };
+
+    // Specify which fields the TimePart should apply too
+    const dateFields = [{name: 'Birth Date'}, {name: 'Register Date'}];
+
+    // Construct de-identification request to be sent by client.
+    const request = {
+      parent: `projects/${projectId}/locations/global`,
+      deidentifyConfig: {
+        recordTransformations: {
+          fieldTransformations: [
+            {
+              fields: dateFields,
+              primitiveTransformation,
+            },
+          ],
+        },
+      },
+      item: {
+        table: tablularData,
+      },
+    };
+
+    // Use the client to send the API request.
+    const [response] = await dlp.deidentifyContent(request);
+
+    // Print results.
+    console.log(
+      `Table after de-identification: ${JSON.stringify(response.item.table)}`
+    );
+  }
+
+  deidentifyWithTimeExtraction();
+  // [END dlp_deidentify_time_extract]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+
+main(...process.argv.slice(2));

dlp/system-test/deid.test.js

@@ -333,4 +333,28 @@ describe('deid', () => {
     }
     assert.include(output, 'INVALID_ARGUMENT');
   });
+
+  // dlp_deidentify_time_extract
+  it('should replace sensitive data in a string using time extraction', () => {
+    let output;
+    try {
+      output = execSync(`node deidentifyWithTimeExtraction.js ${projectId}`);
+    } catch (err) {
+      output = err.message;
+    }
+    assert.match(output, /"stringValue":"1970"/);
+    assert.match(output, /"stringValue":"1996"/);
+    assert.match(output, /"stringValue":"1988"/);
+    assert.match(output, /"stringValue":"2001"/);
+  });
+
+  it('should handle deidentification errors', () => {
+    let output;
+    try {
+      output = execSync('node deidentifyWithTimeExtraction.js BAD_PROJECT_ID');
+    } catch (err) {
+      output = err.message;
+    }
+    assert.include(output, 'INVALID_ARGUMENT');
+  });
 });

dlp/system-test/metadata.test.js

@@ -17,17 +17,42 @@
 const {assert} = require('chai');
 const {describe, it, before} = require('mocha');
 const cp = require('child_process');
+const uuid = require('uuid');
 const DLP = require('@google-cloud/dlp');
 
+const dataProject = 'bigquery-public-data';
+const dataSetId = 'samples';
+const tableId = 'github_nested';
+const fieldId = 'url';
+
+const bucketName = process.env.BUCKET_NAME;
+
 const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
 
 const client = new DLP.DlpServiceClient();
 describe('metadata', () => {
-  let projectId;
+  let projectId, storedInfoTypeId;
 
   before(async () => {
     projectId = await client.getProjectId();
   });
+
+  // Delete stored infotypes created in the snippets.
+  afterEach(async () => {
+    if (!storedInfoTypeId) {
+      return;
+    }
+    const request = {
+      name: storedInfoTypeId,
+    };
+    try {
+      await client.deleteStoredInfoType(request);
+      storedInfoTypeId = '';
+    } catch (err) {
+      throw `Error in deleting store infoType: ${err.message || err}`;
+    }
+  });
+
   it('should list info types', () => {
     const output = execSync(`node metadata.js ${projectId} infoTypes`);
     assert.match(output, /US_DRIVERS_LICENSE_NUMBER/);
@@ -39,4 +64,29 @@ describe('metadata', () => {
     );
     assert.notMatch(output, /US_DRIVERS_LICENSE_NUMBER/);
   });
+
+  // dlp_create_stored_infotype
+  it('should create a stored infotype', () => {
+    const infoTypeId = `stored-infoType-${uuid.v4()}`;
+    const infoTypeOutputPath = `gs://${bucketName}`;
+    const output = execSync(
+      `node createStoredInfoType.js ${projectId} ${infoTypeId} ${infoTypeOutputPath} ${dataProject} ${dataSetId} ${tableId} ${fieldId}`
+    );
+    assert.match(output, /InfoType stored successfully:/);
+    storedInfoTypeId = output.split(':')[1].trim();
+  });
+
+  it('should handle stored infotype creation errors', () => {
+    let output;
+    const infoTypeId = `stored-infoType-${uuid.v4()}`;
+    const infoTypeOutputPath = 'INFOTYPE_OUTPUT_PATH';
+    try {
+      output = execSync(
+        `node createStoredInfoType.js BAD_PROJECT_ID ${infoTypeId} ${infoTypeOutputPath} ${dataProject} ${dataSetId} ${tableId} ${fieldId}`
+      );
+    } catch (err) {
+      output = err.message;
+    }
+    assert.include(output, 'INVALID_ARGUMENT');
+  });
 });