From 7ace33d4b4bc48a3bae9ef2ae19873c5f830468e Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 13:55:44 -0700 Subject: [PATCH 01/21] feat(samples): Add POSIX samples --- samples/posix-request.js | 87 ++++++++++++++++++++++++++++++ samples/test/posix-request.test.js | 66 +++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 samples/posix-request.js create mode 100644 samples/test/posix-request.test.js diff --git a/samples/posix-request.js b/samples/posix-request.js new file mode 100644 index 0000000..02a661d --- /dev/null +++ b/samples/posix-request.js @@ -0,0 +1,87 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + rootDirectory = '', + gcsSinkBucket = '' +) { + // [START storagetransfer_transfer_from_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The root directory path on the filesystem + // const rootDirectory = '/directory/to/transfer', + + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = 'my-sink-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + posixDataSource: { + rootDirectory, + }, + gcsDataSink: {bucketName: gcsSinkBucket}, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' with name '${transferJob.name}'` + ); + } + + transferDirectory(); + // [END storagetransfer_transfer_from_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js new file mode 100644 index 0000000..16f94da --- /dev/null +++ b/samples/test/posix-request.test.js @@ -0,0 +1,66 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs/promises'); +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let gcsSinkBucket; + let tempDirectory; + let tempFile; + + before(async () => { + tempDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-request-test-') + ); + tempFile = path.join(tempDirectory, 'text.txt'); + + await fs.writeFile(tempFile, 'test data'); + + projectId = await testBucketManager.getProjectId(); + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.rm(tempFile); + await fs.rmdir(tempDirectory); + }); + + it('should create a transfer job from POSIX to GCS', async () => { + const output = await runSample('posix-request', [ + projectId, + tempDirectory, + gcsSinkBucket, + ]); + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); From 612b850fd2924cde83a3df611ae4ce843bdbefd3 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 26 May 2022 20:57:49 +0000 Subject: [PATCH 02/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index 0acc02c..1bdc4fc 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Check-latest-transfer-operation | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/check-latest-transfer-operation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/check-latest-transfer-operation.js,samples/README.md) | | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | +| Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | | Transfer-check | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/transfer-check.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/transfer-check.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index a8d35f2..53d2c2e 100644 --- a/samples/README.md +++ b/samples/README.md @@ -16,6 +16,7 @@ * [Check-latest-transfer-operation](#check-latest-transfer-operation) * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) * [Nearline-request](#nearline-request) + * [Posix-request](#posix-request) * [Quickstart](#quickstart) * [Transfer-check](#transfer-check) @@ -102,6 +103,23 @@ __Usage:__ +### Posix-request + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) + +__Usage:__ + + +`node samples/posix-request.js` + + +----- + + + + ### Quickstart View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/quickstart.js). From 7cc13dd8de9f6ace87c76f2ba3e9b83d90f7fc17 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 14:40:36 -0700 Subject: [PATCH 03/21] Fix: `fs.promises` import --- samples/test/posix-request.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index 16f94da..62db3b3 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -16,7 +16,7 @@ 'use strict'; -const fs = require('fs/promises'); +const fs = require('fs').promises; const os = require('os'); const path = require('path'); From e1696d1065fc4d1a9eb2119866b7158befec2c7d Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 16:29:44 -0700 Subject: [PATCH 04/21] fix: clean-up created transfer job --- samples/test/posix-request.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index 62db3b3..a01f67f 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -60,6 +60,12 @@ describe('posix-request', () => { gcsSinkBucket, ]); + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + // Find at least 1 transfer operation from the transfer job in the output assert.include(output, 'Created and ran a transfer job'); }); From 095198323661d9109728998051a9354a92eaa9f4 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 16:43:03 -0700 Subject: [PATCH 05/21] chore: Add (safe) debug log --- samples/test/posix-request.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index a01f67f..f3f3806 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -54,6 +54,11 @@ describe('posix-request', () => { }); it('should create a transfer job from POSIX to GCS', async () => { + console.log( + 'ggsa', + await testTransferJobManager.client.getGoogleServiceAccount() + ); + const output = await runSample('posix-request', [ projectId, tempDirectory, From 4a47cd5692ac784ada33f8a22e683d8ca003335a Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 16:50:23 -0700 Subject: [PATCH 06/21] fix: misc bugs --- samples/test/posix-request.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index f3f3806..7fcd771 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -49,14 +49,14 @@ describe('posix-request', () => { after(async () => { await testBucketManager.deleteBuckets(); await testTransferJobManager.cleanUp(); - await fs.rm(tempFile); + await fs.unlink(tempFile); await fs.rmdir(tempDirectory); }); it('should create a transfer job from POSIX to GCS', async () => { console.log( 'ggsa', - await testTransferJobManager.client.getGoogleServiceAccount() + await testTransferJobManager.client.getGoogleServiceAccount({projectId}) ); const output = await runSample('posix-request', [ From 6236e9ec98f0c2e5b6257604fc8d977e6d2c9f74 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 26 May 2022 17:10:21 -0700 Subject: [PATCH 07/21] chore: remove debug log --- samples/test/posix-request.test.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index 7fcd771..99ce0ec 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -54,11 +54,6 @@ describe('posix-request', () => { }); it('should create a transfer job from POSIX to GCS', async () => { - console.log( - 'ggsa', - await testTransferJobManager.client.getGoogleServiceAccount({projectId}) - ); - const output = await runSample('posix-request', [ projectId, tempDirectory, From 86301e0c65f4985d35e0f2589300e56e51c22c19 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Fri, 27 May 2022 15:30:55 -0700 Subject: [PATCH 08/21] feat: add POSIX to POSIX sample - Also, update tests and comments --- samples/posix-request.js | 11 +- samples/posix-to-posix-request.js | 106 ++++++++++++++++++++ samples/test/posix-request.test.js | 21 ++-- samples/test/posix-to-posix-request.test.js | 89 ++++++++++++++++ 4 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 samples/posix-to-posix-request.js create mode 100644 samples/test/posix-to-posix-request.test.js diff --git a/samples/posix-request.js b/samples/posix-request.js index 02a661d..0f2301d 100644 --- a/samples/posix-request.js +++ b/samples/posix-request.js @@ -18,6 +18,7 @@ async function main( projectId = 'my-project', + sourceAgentPoolName = '', rootDirectory = '', gcsSinkBucket = '' ) { @@ -34,8 +35,11 @@ async function main( // Your project id // const projectId = 'my-project' - // The root directory path on the filesystem - // const rootDirectory = '/directory/to/transfer', + // The agent pool associated with the POSIX data source. Defaults to the default agent. + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', // The ID of the GCS bucket to transfer data to // const gcsSinkBucket = 'my-sink-bucket' @@ -51,6 +55,7 @@ async function main( transferJob: { projectId, transferSpec: { + sourceAgentPoolName, posixDataSource: { rootDirectory, }, @@ -71,7 +76,7 @@ async function main( await client.runTransferJob(runRequest); console.log( - `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' with name '${transferJob.name}'` + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' with name ${transferJob.name}` ); } diff --git a/samples/posix-to-posix-request.js b/samples/posix-to-posix-request.js new file mode 100644 index 0000000..4a1121b --- /dev/null +++ b/samples/posix-to-posix-request.js @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = '', + sinkAgentPoolName = '', + rootDirectory = '', + destinationDirectory = '', + bucketName = '' +) { + // [START storagetransfer_transfer_posix_to_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data source. Defaults to the default agent. + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The agent pool associated with the POSIX data sink. Defaults to the default agent. + // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // The root directory path on the sink filesystem + // const destinationDirectory = '/directory/to/transfer/sink' + + // The ID of the GCS bucket for intermediate storage The intermediate Cloud Storage intermediate data location. + // const bucketName = 'my-intermediate-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + sinkAgentPoolName, + posixDataSource: { + rootDirectory, + }, + posixDataSink: { + rootDirectory: destinationDirectory, + }, + gcsIntermediateDataLocation: { + bucketName, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${destinationDirectory}' with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_transfer_posix_to_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index 99ce0ec..77b7ef6 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -30,19 +30,25 @@ describe('posix-request', () => { const testTransferJobManager = new TransferJobManager(); let projectId; + let sourceAgentPoolName; + let rootDirectory; let gcsSinkBucket; - let tempDirectory; + let tempFile; before(async () => { - tempDirectory = await fs.mkdtemp( - path.join(os.tmpdir(), 'sts-posix-request-test-') + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-request-test-src-') ); - tempFile = path.join(tempDirectory, 'text.txt'); + tempFile = path.join(rootDirectory, 'text.txt'); await fs.writeFile(tempFile, 'test data'); - projectId = await testBucketManager.getProjectId(); gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; }); @@ -50,13 +56,14 @@ describe('posix-request', () => { await testBucketManager.deleteBuckets(); await testTransferJobManager.cleanUp(); await fs.unlink(tempFile); - await fs.rmdir(tempDirectory); + await fs.rmdir(rootDirectory); }); it('should create a transfer job from POSIX to GCS', async () => { const output = await runSample('posix-request', [ projectId, - tempDirectory, + sourceAgentPoolName, + rootDirectory, gcsSinkBucket, ]); diff --git a/samples/test/posix-to-posix-request.test.js b/samples/test/posix-to-posix-request.test.js new file mode 100644 index 0000000..7fde0be --- /dev/null +++ b/samples/test/posix-to-posix-request.test.js @@ -0,0 +1,89 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-to-posix-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceAgentPoolName; + let sinkAgentPoolName; + let rootDirectory; + let destinationDirectory; + let bucketName; + + let tempFile; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + sinkAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-src-') + ); + tempFile = path.join(rootDirectory, 'text.txt'); + + destinationDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-sink-') + ); + + await fs.writeFile(tempFile, 'test data'); + + bucketName = (await testBucketManager.generateGCSBucket()).name; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + + await fs.unlink(tempFile); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job from POSIX to POSIX', async () => { + const output = await runSample('posix-to-posix-request', [ + projectId, + sourceAgentPoolName, + sinkAgentPoolName, + rootDirectory, + destinationDirectory, + bucketName, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); From 1946b0ce0687a11d16ac3349a7be5cce93c494fa Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Fri, 27 May 2022 15:34:13 -0700 Subject: [PATCH 09/21] chore: typo & clean-up --- samples/posix-request.js | 2 +- samples/posix-to-posix-request.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/posix-request.js b/samples/posix-request.js index 0f2301d..8ff8ee6 100644 --- a/samples/posix-request.js +++ b/samples/posix-request.js @@ -35,7 +35,7 @@ async function main( // Your project id // const projectId = 'my-project' - // The agent pool associated with the POSIX data source. Defaults to the default agent. + // The agent pool associated with the POSIX data source. Defaults to the default agent // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' // The root directory path on the source filesystem diff --git a/samples/posix-to-posix-request.js b/samples/posix-to-posix-request.js index 4a1121b..50f3097 100644 --- a/samples/posix-to-posix-request.js +++ b/samples/posix-to-posix-request.js @@ -37,10 +37,10 @@ async function main( // Your project id // const projectId = 'my-project' - // The agent pool associated with the POSIX data source. Defaults to the default agent. + // The agent pool associated with the POSIX data source. Defaults to the default agent // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' - // The agent pool associated with the POSIX data sink. Defaults to the default agent. + // The agent pool associated with the POSIX data sink. Defaults to the default agent // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' // The root directory path on the source filesystem @@ -49,7 +49,7 @@ async function main( // The root directory path on the sink filesystem // const destinationDirectory = '/directory/to/transfer/sink' - // The ID of the GCS bucket for intermediate storage The intermediate Cloud Storage intermediate data location. + // The ID of the GCS bucket for intermediate storage // const bucketName = 'my-intermediate-bucket' // Creates a client From 7215ff643f99b26d625d8e927837b0071bf25475 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 27 May 2022 22:35:46 +0000 Subject: [PATCH 10/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index 1bdc4fc..e70d913 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | | Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | +| Posix-to-posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-to-posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-to-posix-request.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | | Transfer-check | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/transfer-check.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/transfer-check.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index 53d2c2e..1372641 100644 --- a/samples/README.md +++ b/samples/README.md @@ -17,6 +17,7 @@ * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) * [Nearline-request](#nearline-request) * [Posix-request](#posix-request) + * [Posix-to-posix-request](#posix-to-posix-request) * [Quickstart](#quickstart) * [Transfer-check](#transfer-check) @@ -120,6 +121,23 @@ __Usage:__ +### Posix-to-posix-request + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-to-posix-request.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-to-posix-request.js,samples/README.md) + +__Usage:__ + + +`node samples/posix-to-posix-request.js` + + +----- + + + + ### Quickstart View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/quickstart.js). From cbf6aae163294e8f77220c97a65c567c7f0d4752 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 31 May 2022 19:51:08 -0700 Subject: [PATCH 11/21] refactor: styling --- samples/test/posix-request.test.js | 6 +++--- samples/test/posix-to-posix-request.test.js | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/test/posix-request.test.js b/samples/test/posix-request.test.js index 77b7ef6..d7e0d11 100644 --- a/samples/test/posix-request.test.js +++ b/samples/test/posix-request.test.js @@ -45,11 +45,11 @@ describe('posix-request', () => { rootDirectory = await fs.mkdtemp( path.join(os.tmpdir(), 'sts-posix-request-test-src-') ); - tempFile = path.join(rootDirectory, 'text.txt'); - - await fs.writeFile(tempFile, 'test data'); gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); }); after(async () => { diff --git a/samples/test/posix-to-posix-request.test.js b/samples/test/posix-to-posix-request.test.js index 7fde0be..da7fd9a 100644 --- a/samples/test/posix-to-posix-request.test.js +++ b/samples/test/posix-to-posix-request.test.js @@ -48,15 +48,15 @@ describe('posix-to-posix-request', () => { rootDirectory = await fs.mkdtemp( path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-src-') ); - tempFile = path.join(rootDirectory, 'text.txt'); destinationDirectory = await fs.mkdtemp( path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-sink-') ); - await fs.writeFile(tempFile, 'test data'); - bucketName = (await testBucketManager.generateGCSBucket()).name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); }); after(async () => { From ba86cae4db602ce1f486f622cf9388bc7be316da Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Fri, 3 Jun 2022 12:34:47 -0700 Subject: [PATCH 12/21] feat: Add POSIX Download sample --- samples/posix-download.js | 99 +++++++++++++++++++++++++++++ samples/test/posix-download.test.js | 88 +++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 samples/posix-download.js create mode 100644 samples/test/posix-download.test.js diff --git a/samples/posix-download.js b/samples/posix-download.js new file mode 100644 index 0000000..448847b --- /dev/null +++ b/samples/posix-download.js @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sinkAgentPoolName = '', + gcsSourceBucket = '', + gcsSourceBucketPath = '', + rootDirectory = '' +) { + // [START storagetransfer_download_to_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data sink. Defaults to the default agent + // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // Google Cloud Storage source bucket name + // const gcsSourceBucket = 'my-gcs-source-bucket' + + // An optional path to on the Google Cloud Storage bucket to download from + // const gcsSourceBucketPath = 'foo/bar/' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sinkAgentPoolName, + gcsDataSource: { + bucketName: gcsSourceBucket, + path: gcsSourceBucketPath, + }, + posixDataSink: { + rootDirectory, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourceBucketPath}\`) to '${rootDirectory}' with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_download_to_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/samples/test/posix-download.test.js b/samples/test/posix-download.test.js new file mode 100644 index 0000000..8286bd4 --- /dev/null +++ b/samples/test/posix-download.test.js @@ -0,0 +1,88 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-download', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sinkAgentPoolName; + let gcsSourceBucket; + let gcsSourceBucketPath; + let rootDirectory; + + let tempObject; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sinkAgentPoolName = ''; + + const bucket = await testBucketManager.generateGCSBucket(); + gcsSourceBucket = bucket.name; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-download-test-sink-') + ); + + gcsSourceBucketPath = rootDirectory; + + tempObject = bucket.file(path.join(rootDirectory, 'text.txt')); + + await tempObject.save('test data'); + }); + + after(async () => { + await tempObject.delete(); + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job from GCS to POSIX', async () => { + const output = await runSample('posix-download', [ + projectId, + sinkAgentPoolName, + gcsSourceBucket, + gcsSourceBucketPath, + rootDirectory, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include( + output, + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourceBucketPath}\`) to '${rootDirectory}'` + ); + }); +}); From d613f20b7857e113876874a62d08a2fca6e093d8 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 3 Jun 2022 19:36:25 +0000 Subject: [PATCH 13/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index e70d913..f592dca 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Check-latest-transfer-operation | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/check-latest-transfer-operation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/check-latest-transfer-operation.js,samples/README.md) | | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | +| Posix-download | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-download.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-download.js,samples/README.md) | | Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | | Posix-to-posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-to-posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-to-posix-request.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index 1372641..cac0e84 100644 --- a/samples/README.md +++ b/samples/README.md @@ -16,6 +16,7 @@ * [Check-latest-transfer-operation](#check-latest-transfer-operation) * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) * [Nearline-request](#nearline-request) + * [Posix-download](#posix-download) * [Posix-request](#posix-request) * [Posix-to-posix-request](#posix-to-posix-request) * [Quickstart](#quickstart) @@ -104,6 +105,23 @@ __Usage:__ +### Posix-download + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-download.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-download.js,samples/README.md) + +__Usage:__ + + +`node samples/posix-download.js` + + +----- + + + + ### Posix-request View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js). From 9b082577e68f300661a3c19df9eb05d1d1464506 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Fri, 3 Jun 2022 12:50:20 -0700 Subject: [PATCH 14/21] fix: Add ending '/' --- samples/test/posix-download.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/test/posix-download.test.js b/samples/test/posix-download.test.js index 8286bd4..0b5b3a8 100644 --- a/samples/test/posix-download.test.js +++ b/samples/test/posix-download.test.js @@ -50,7 +50,8 @@ describe('posix-download', () => { path.join(os.tmpdir(), 'sts-posix-download-test-sink-') ); - gcsSourceBucketPath = rootDirectory; + // API requires path to end with '/' + gcsSourceBucketPath = rootDirectory + path.posix.sep; tempObject = bucket.file(path.join(rootDirectory, 'text.txt')); From dc22f572ff8fead402c345a38fa0e5d91d762ce0 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Fri, 3 Jun 2022 12:51:06 -0700 Subject: [PATCH 15/21] style: shorten variable --- samples/posix-download.js | 10 +++++----- samples/test/posix-download.test.js | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/posix-download.js b/samples/posix-download.js index 448847b..818de46 100644 --- a/samples/posix-download.js +++ b/samples/posix-download.js @@ -20,7 +20,7 @@ async function main( projectId = 'my-project', sinkAgentPoolName = '', gcsSourceBucket = '', - gcsSourceBucketPath = '', + gcsSourcePath = '', rootDirectory = '' ) { // [START storagetransfer_download_to_posix] @@ -42,8 +42,8 @@ async function main( // Google Cloud Storage source bucket name // const gcsSourceBucket = 'my-gcs-source-bucket' - // An optional path to on the Google Cloud Storage bucket to download from - // const gcsSourceBucketPath = 'foo/bar/' + // An optional path on the Google Cloud Storage bucket to download from + // const gcsSourcePath = 'foo/bar/' // The root directory path on the source filesystem // const rootDirectory = '/directory/to/transfer/source', @@ -62,7 +62,7 @@ async function main( sinkAgentPoolName, gcsDataSource: { bucketName: gcsSourceBucket, - path: gcsSourceBucketPath, + path: gcsSourcePath, }, posixDataSink: { rootDirectory, @@ -83,7 +83,7 @@ async function main( await client.runTransferJob(runRequest); console.log( - `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourceBucketPath}\`) to '${rootDirectory}' with name ${transferJob.name}` + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourcePath}\`) to '${rootDirectory}' with name ${transferJob.name}` ); } diff --git a/samples/test/posix-download.test.js b/samples/test/posix-download.test.js index 0b5b3a8..bfe904f 100644 --- a/samples/test/posix-download.test.js +++ b/samples/test/posix-download.test.js @@ -32,7 +32,7 @@ describe('posix-download', () => { let projectId; let sinkAgentPoolName; let gcsSourceBucket; - let gcsSourceBucketPath; + let gcsSourcePath; let rootDirectory; let tempObject; @@ -51,7 +51,7 @@ describe('posix-download', () => { ); // API requires path to end with '/' - gcsSourceBucketPath = rootDirectory + path.posix.sep; + gcsSourcePath = rootDirectory + path.posix.sep; tempObject = bucket.file(path.join(rootDirectory, 'text.txt')); @@ -70,7 +70,7 @@ describe('posix-download', () => { projectId, sinkAgentPoolName, gcsSourceBucket, - gcsSourceBucketPath, + gcsSourcePath, rootDirectory, ]); @@ -83,7 +83,7 @@ describe('posix-download', () => { // Find at least 1 transfer operation from the transfer job in the output assert.include( output, - `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourceBucketPath}\`) to '${rootDirectory}'` + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourcePath}\`) to '${rootDirectory}'` ); }); }); From bc80220e79e703aac4df8b3a099f344e56394327 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 14 Jun 2022 13:30:21 -0700 Subject: [PATCH 16/21] feat: Transfer Manifest request --- samples/manifest-request.js | 99 +++++++++++++++++++++++++++ samples/test/manifest-request.test.js | 93 +++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 samples/manifest-request.js create mode 100644 samples/test/manifest-request.test.js diff --git a/samples/manifest-request.js b/samples/manifest-request.js new file mode 100644 index 0000000..ade6a52 --- /dev/null +++ b/samples/manifest-request.js @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = '', + rootDirectory = '', + gcsSinkBucket = '', + manifestLocation = '' +) { + // [START storagetransfer_manifest_request] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data source. Defaults to the default agent + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = 'my-sink-bucket' + + // Transfer manifest location. Must be a + // const manifestLocation = 'gs://my-bucket/sample_manifest.csv' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + posixDataSource: { + rootDirectory, + }, + gcsDataSink: {bucketName: gcsSinkBucket}, + transferManifest: { + location: manifestLocation, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' using manifest `${}` with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_manifest_request] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/samples/test/manifest-request.test.js b/samples/test/manifest-request.test.js new file mode 100644 index 0000000..e7a4ffa --- /dev/null +++ b/samples/test/manifest-request.test.js @@ -0,0 +1,93 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('manifest-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceAgentPoolName; + let rootDirectory; + let gcsSinkBucket; + let manifestLocation; + + let tempManifestObject; + let tempFile; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-manifest-request-test-src-') + ); + + const bucket = await testBucketManager.generateGCSBucket(); + + gcsSinkBucket = bucket.name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); + + // Double-quote to escape double-quotes in CSV text + const csvContent = `"${tempFile.replaceAll('"', '""')}"`; + + tempManifestObject = bucket.file('manifest.csv'); + await tempManifestObject.save(csvContent); + + manifestLocation = `gs://${bucket.name}/${tempManifestObject.name}`; + }); + + after(async () => { + await tempManifestObject.delete(); + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.unlink(tempFile); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job with a manifest from POSIX to GCS', async () => { + const output = await runSample('manifest-request', [ + projectId, + sourceAgentPoolName, + rootDirectory, + gcsSinkBucket, + manifestLocation, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); From ab5dafa72215cb0e944607d570d2daeeb43c176d Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 14 Jun 2022 13:30:50 -0700 Subject: [PATCH 17/21] fix: typo --- samples/manifest-request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manifest-request.js b/samples/manifest-request.js index ade6a52..3817bfd 100644 --- a/samples/manifest-request.js +++ b/samples/manifest-request.js @@ -83,7 +83,7 @@ async function main( await client.runTransferJob(runRequest); console.log( - `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' using manifest `${}` with name ${transferJob.name}` + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' using manifest \`${manifestLocation}\` with name ${transferJob.name}` ); } From 28bb7cf0adec05e9c6ab75b44e8a21e45afdad63 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 14 Jun 2022 20:32:00 +0000 Subject: [PATCH 18/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index f592dca..1d5221b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Aws-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/aws-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/aws-request.js,samples/README.md) | | Check-latest-transfer-operation | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/check-latest-transfer-operation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/check-latest-transfer-operation.js,samples/README.md) | | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | +| Manifest-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | | Posix-download | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-download.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-download.js,samples/README.md) | | Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index cac0e84..732d750 100644 --- a/samples/README.md +++ b/samples/README.md @@ -15,6 +15,7 @@ * [Aws-request](#aws-request) * [Check-latest-transfer-operation](#check-latest-transfer-operation) * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) + * [Manifest-request](#manifest-request) * [Nearline-request](#nearline-request) * [Posix-download](#posix-download) * [Posix-request](#posix-request) @@ -88,6 +89,23 @@ __Usage:__ +### Manifest-request + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) + +__Usage:__ + + +`node samples/manifest-request.js` + + +----- + + + + ### Nearline-request View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js). From 9b199ace46a7548e5a4466dd46239ffea434bf5b Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 14 Jun 2022 20:32:29 +0000 Subject: [PATCH 19/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index f592dca..1d5221b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Aws-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/aws-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/aws-request.js,samples/README.md) | | Check-latest-transfer-operation | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/check-latest-transfer-operation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/check-latest-transfer-operation.js,samples/README.md) | | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | +| Manifest-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | | Posix-download | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-download.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-download.js,samples/README.md) | | Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index cac0e84..732d750 100644 --- a/samples/README.md +++ b/samples/README.md @@ -15,6 +15,7 @@ * [Aws-request](#aws-request) * [Check-latest-transfer-operation](#check-latest-transfer-operation) * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) + * [Manifest-request](#manifest-request) * [Nearline-request](#nearline-request) * [Posix-download](#posix-download) * [Posix-request](#posix-request) @@ -88,6 +89,23 @@ __Usage:__ +### Manifest-request + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) + +__Usage:__ + + +`node samples/manifest-request.js` + + +----- + + + + ### Nearline-request View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js). From 233441c4c005086147e22cfd473f7f31aa73c5ab Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 14 Jun 2022 20:33:17 +0000 Subject: [PATCH 20/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 + samples/README.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/README.md b/README.md index f592dca..1d5221b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage-tra | Aws-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/aws-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/aws-request.js,samples/README.md) | | Check-latest-transfer-operation | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/check-latest-transfer-operation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/check-latest-transfer-operation.js,samples/README.md) | | Get-transfer-job-with-retries | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/get-transfer-job-with-retries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/get-transfer-job-with-retries.js,samples/README.md) | +| Manifest-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) | | Nearline-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/nearline-request.js,samples/README.md) | | Posix-download | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-download.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-download.js,samples/README.md) | | Posix-request | [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/posix-request.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/posix-request.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index cac0e84..732d750 100644 --- a/samples/README.md +++ b/samples/README.md @@ -15,6 +15,7 @@ * [Aws-request](#aws-request) * [Check-latest-transfer-operation](#check-latest-transfer-operation) * [Get-transfer-job-with-retries](#get-transfer-job-with-retries) + * [Manifest-request](#manifest-request) * [Nearline-request](#nearline-request) * [Posix-download](#posix-download) * [Posix-request](#posix-request) @@ -88,6 +89,23 @@ __Usage:__ +### Manifest-request + +View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/manifest-request.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage-transfer&page=editor&open_in_editor=samples/manifest-request.js,samples/README.md) + +__Usage:__ + + +`node samples/manifest-request.js` + + +----- + + + + ### Nearline-request View the [source code](https://github.com/googleapis/nodejs-storage-transfer/blob/main/samples/nearline-request.js). From 87c90a28a9993f0e4f856abd1f47af78de908243 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 14 Jun 2022 14:46:18 -0700 Subject: [PATCH 21/21] fix: `replaceAll` (not available in Node 12) -> `replace` --- samples/test/manifest-request.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/test/manifest-request.test.js b/samples/test/manifest-request.test.js index e7a4ffa..b0db4a4 100644 --- a/samples/test/manifest-request.test.js +++ b/samples/test/manifest-request.test.js @@ -56,7 +56,7 @@ describe('manifest-request', () => { await fs.writeFile(tempFile, 'test data'); // Double-quote to escape double-quotes in CSV text - const csvContent = `"${tempFile.replaceAll('"', '""')}"`; + const csvContent = `"${tempFile.replace(/"/g, '""')}"`; tempManifestObject = bucket.file('manifest.csv'); await tempManifestObject.save(csvContent);