From 9651d73cdb87e6fc24cafb5727286977504e4054 Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Mon, 30 Oct 2017 10:20:41 -0700 Subject: [PATCH] Cloud Tasks samples. --- appengine/cloudtasks/README.md | 87 ++++++++ appengine/cloudtasks/app.yaml | 2 + appengine/cloudtasks/createTask.js | 129 ++++++++++++ appengine/cloudtasks/package.json | 45 +++++ appengine/cloudtasks/server.js | 47 +++++ appengine/cloudtasks/test/createTask.test.js | 59 ++++++ circle.yml | 1 + cloudtasks/README.md | 76 +++++++ cloudtasks/package.json | 36 ++++ cloudtasks/system-test/tasks.test.js | 73 +++++++ cloudtasks/tasks.js | 198 +++++++++++++++++++ 11 files changed, 753 insertions(+) create mode 100644 appengine/cloudtasks/README.md create mode 100644 appengine/cloudtasks/app.yaml create mode 100644 appengine/cloudtasks/createTask.js create mode 100644 appengine/cloudtasks/package.json create mode 100644 appengine/cloudtasks/server.js create mode 100644 appengine/cloudtasks/test/createTask.test.js create mode 100644 cloudtasks/README.md create mode 100644 cloudtasks/package.json create mode 100644 cloudtasks/system-test/tasks.test.js create mode 100644 cloudtasks/tasks.js diff --git a/appengine/cloudtasks/README.md b/appengine/cloudtasks/README.md new file mode 100644 index 0000000000..ab2db06d66 --- /dev/null +++ b/appengine/cloudtasks/README.md @@ -0,0 +1,87 @@ +# Node.js Google Cloud Tasks sample for Google App Engine + +This sample demonstrates how to use [Google Cloud Tasks](https://cloud.google.com/cloud-tasks/) +on [Google App Engine Flexible Environment](https://cloud.google.com/appengine/docs/flexible/nodejs). + +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. + +`createTask.js` is a simple command-line program to create tasks to be pushed to +the App Engine app. + +`server.js` is the main App Engine app. This app serves as an endpoint to +receive App Engine task attempts. + +`app.yaml` configures the App Engine app. + +* [Setup](#setup) +* [Running locally](#running-locally) +* [Deploying to App Engine](#deploying-to-app-engine) +* [Running the tests](#running-the-tests) + +## Setup + +Before you can run or deploy the sample, you need to do the following: + +1. Refer to the [appengine/README.md][readme] file for instructions on + running and deploying. +1. Enable the Cloud Tasks API in the [Google Cloud Console](https://cloud.google.com/apis/library/cloudtasks.googleapis.com). +1. Install dependencies: + + With `npm`: + + npm install + + or with `yarn`: + + yarn install + +## Creating a queue + +To create a queue using the Cloud SDK, use the following gcloud command: + + gcloud alpha tasks queues create-app-engine-queue my-appengine-queue + +Note: A newly created queue will route to the default App Engine service and +version unless configured to do otherwise. Read the online help for the +`create-app-engine-queue` or the `update-app-engine-queue` commands to learn +about routing overrides for App Engine queues. + +## Deploying the App Engine app + +Deploy the App Engine app with gcloud: + + gcloud app deploy + +Verify the index page is serving: + + gcloud app browse + +The App Engine app serves as a target for the push requests. It has an +endpoint `/log_payload` that reads the payload (i.e., the request body) of the +HTTP POST request and logs it. The log output can be viewed with: + + gcloud app logs read + +## Running the Samples + +To get usage information: `node createTask.js --help` + +Which prints: + +``` +Options: + --version Show version number [boolean] + --location, -l Location of the queue to add the task to. [string] [required] + --queue, -q ID (short name) of the queue to add the task to. [string] [required] + --project, -p Project of the queue to add the task to. [string] [required] + --payload, -d (Optional) Payload to attach to the push queue. [string] + --inSeconds, -s (Optional) The number of seconds from now to schedule task attempt. [number] + --help Show help [boolean] + +Examples: + node createTask.js --project my-project-id + +For more information, see https://cloud.google.com/cloud-tasks +``` diff --git a/appengine/cloudtasks/app.yaml b/appengine/cloudtasks/app.yaml new file mode 100644 index 0000000000..f1774cbf7f --- /dev/null +++ b/appengine/cloudtasks/app.yaml @@ -0,0 +1,2 @@ +env: flex +runtime: nodejs diff --git a/appengine/cloudtasks/createTask.js b/appengine/cloudtasks/createTask.js new file mode 100644 index 0000000000..ce34c5fb87 --- /dev/null +++ b/appengine/cloudtasks/createTask.js @@ -0,0 +1,129 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const google = require('googleapis'); +const cloudtasks = google.cloudtasks('v2beta2'); + +function authorize (callback) { + google.auth.getApplicationDefault(function (err, authClient) { + if (err) { + console.error('authentication failed: ', err); + return; + } + if (authClient.createScopedRequired && authClient.createScopedRequired()) { + var scopes = ['https://www.googleapis.com/auth/cloud-platform']; + authClient = authClient.createScoped(scopes); + } + callback(authClient); + }); +} + +/** + * Create a task for a given queue with an arbitrary payload. + */ +function createTask (project, location, queue, options) { + authorize((authClient) => { + const task = { + app_engine_http_request: { + http_method: 'POST', + relative_url: '/log_payload' + } + }; + + if (options.payload !== undefined) { + task.app_engine_http_request.payload = Buffer.from(options.payload).toString('base64'); + } + + if (options.inSeconds !== undefined) { + task.schedule_time = (new Date(options.inSeconds * 1000 + Date.now())).toISOString(); + } + + const request = { + parent: `projects/${project}/locations/${location}/queues/${queue}`, // TODO: Update placeholder value. + resource: { + task: task + }, + auth: authClient + }; + + console.log('Sending task %j', task); + + cloudtasks.projects.locations.queues.tasks.create(request, (err, response) => { + if (err) { + console.error(err); + return; + } + + console.log('Created task.', response.name); + console.log(JSON.stringify(response, null, 2)); + }); + }); +} + +const cli = require(`yargs`) + .options({ + location: { + alias: 'l', + description: 'Location of the queue to add the task to.', + type: 'string', + requiresArg: true, + required: true + }, + queue: { + alias: 'q', + description: 'ID (short name) of the queue to add the task to.', + type: 'string', + requiresArg: true, + required: true + }, + project: { + alias: 'p', + description: 'Project of the queue to add the task to.', + default: process.env.GCLOUD_PROJECT, + type: 'string', + requiresArg: true, + required: true + }, + payload: { + alias: 'd', + description: '(Optional) Payload to attach to the push queue.', + type: 'string', + requiresArg: true + }, + inSeconds: { + alias: 's', + description: '(Optional) The number of seconds from now to schedule task attempt.', + type: 'number', + requiresArg: true + } + }) + .example(`node $0 --project my-project-id`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/cloud-tasks`) + .strict(); + +if (module === require.main) { + const opts = cli.help().parse(process.argv.slice(2)); + + process.env.GCLOUD_PROJECT = opts.project; + + createTask(opts.project, opts.location, opts.queue, opts); +} + +exports.authorize = authorize; +exports.createTask = createTask; diff --git a/appengine/cloudtasks/package.json b/appengine/cloudtasks/package.json new file mode 100644 index 0000000000..8eb6adfa4d --- /dev/null +++ b/appengine/cloudtasks/package.json @@ -0,0 +1,45 @@ +{ + "name": "appengine-cloudtasks", + "description": "Google App Engine Flexible Environment Cloud Tasks example.", + "version": "0.0.0", + "license": "Apache-2.0", + "author": "Google Inc.", + "private": true, + "repository": "GoogleCloudPlatform/nodejs-docs-samples", + "engines": { + "node": ">=4.0.0" + }, + "scripts": { + "deploy": "gcloud app deploy", + "lint": "repo-tools lint", + "pretest": "npm run lint", + "unit-test": "ava --verbose test/*.test.js", + "system-test": "repo-tools test app --config package.json --config-key cloud-repo-tools", + "all-test": "npm run unit-test && npm run system-test", + "test": "repo-tools test run --cmd npm -- run all-test", + "e2e-test": "repo-tools test deploy --config package.json --config-key cloud-repo-tools" + }, + "dependencies": { + "body-parser": "1.18.2", + "express": "4.16.2", + "googleapis": "22.2.0", + "yargs": "10.0.3" + }, + "devDependencies": { + "@google-cloud/nodejs-repo-tools": "2.1.1", + "proxyquire": "1.8.0", + "sinon": "4.0.2" + }, + "cloud-repo-tools": { + "requiresKeyFile": true, + "requiresProjectId": true, + "test": { + "app": { + "msg": "Hello, world!", + "args": [ + "server.js" + ] + } + } + } +} diff --git a/appengine/cloudtasks/server.js b/appengine/cloudtasks/server.js new file mode 100644 index 0000000000..b271f6e534 --- /dev/null +++ b/appengine/cloudtasks/server.js @@ -0,0 +1,47 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const bodyParser = require('body-parser'); +const express = require('express'); + +const app = express(); +app.enable('trust proxy'); + +app.use(bodyParser.raw()); +app.use(bodyParser.json()); +app.use(bodyParser.text()); + +app.get('/', (req, res, next) => { + // Basic index to verify app is serving + res.send('Hello, world!').end(); +}); + +app.post('/log_payload', (req, res, next) => { + // Log the request payload + console.log('Received task with payload: %s', req.body); + res.send(`Printed task payload: ${req.body}`).end(); +}); + +app.get('*', (req, res) => { + res.send('OK').end(); +}); + +const PORT = process.env.PORT || 8080; +app.listen(process.env.PORT || 8080, () => { + console.log(`App listening on port ${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); diff --git a/appengine/cloudtasks/test/createTask.test.js b/appengine/cloudtasks/test/createTask.test.js new file mode 100644 index 0000000000..1d7fd3e400 --- /dev/null +++ b/appengine/cloudtasks/test/createTask.test.js @@ -0,0 +1,59 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const proxyquire = require(`proxyquire`).noCallThru(); +const sinon = require(`sinon`); +const test = require(`ava`); +const tools = require(`@google-cloud/nodejs-repo-tools`); + +test.before(tools.stubConsole); +test.after.always(tools.restoreConsole); + +test.cb(`should create a task`, (t) => { + const responseMock = { + name: 'foo' + }; + const cloudtasksMock = { + projects: { + locations: { + queues: { + tasks: { + create: sinon.stub().yields(responseMock) + } + } + } + } + }; + const authClientMock = {}; + + const util = proxyquire(`../createTask`, { + googleapis: { + cloudtasks: sinon.stub().returns(cloudtasksMock), + auth: { + getApplicationDefault: sinon.stub().yields(null, authClientMock) + } + } + }); + + util.createTask('p', 'l', 'q', {}); + + setTimeout(() => { + t.true(console.log.called); + t.is(cloudtasksMock.projects.locations.queues.tasks.create.callCount, 1); + t.end(); + }, 500); +}); diff --git a/circle.yml b/circle.yml index 3b293b5691..804bb55540 100644 --- a/circle.yml +++ b/circle.yml @@ -88,6 +88,7 @@ deployment: - node scripts/build "auth" - node scripts/build "appengine/pubsub" - node scripts/build "bigquery" + - export GCP_QUEUE=nodejs-test-queue-do-not-delete; node scripts/build "cloudtasks" - node scripts/build "containerengine/hello-world" - node scripts/build "datastore" - node scripts/build "debugger" diff --git a/cloudtasks/README.md b/cloudtasks/README.md new file mode 100644 index 0000000000..fcd168440b --- /dev/null +++ b/cloudtasks/README.md @@ -0,0 +1,76 @@ +Google Cloud Platform logo + +# Google Cloud Tasks API Node.js Samples + +The [Cloud Tasks API](https://cloud.google.com/cloud-tasks/docs) enables +developers to manage the execution of large numbers of distributed requests. + +## Table of Contents + +* [Setup](#setup) +* [Samples](#samples) + * [Tasks](#tasks) +* [Running the tests](#running-the-tests) + +## Setup + +1. Read [Prerequisites][prereq] and [How to run a sample][run] first. +1. Install dependencies: + + With **npm**: + + npm install + + With **yarn**: + + yarn install + +[prereq]: ../README.md#prerequisites +[run]: ../README.md#how-to-run-a-sample + +## Samples + +### Tasks + +View the [documentation][tasks_0_docs] or the [source code][tasks_0_code]. + +__Usage:__ `node tasks.js --help` + +``` +tasks.js + +Commands: + tasks.js create Create a task. + tasks.js pull Pull a task. + tasks.js acknowledge Acknowledge a task. + +Options: + --version Show version number [boolean] + --help Show help [boolean] + +Examples: + node tasks.js create my-project-id us-central1 my-queue + node tasks.js pull my-project-id us-central1 my-queue + node tasks.js acknowledge + '{"name":"projects/my-project-id/locations/us-central1/queues/my-queue/tasks/1234","scheduleTime":"2017-11-01T22:27: + 53.628279Z"}' + +For more information, see https://cloud.google.com/cloud-tasks/docs +``` + +[tasks_0_docs]: https://cloud.google.com/cloud-tasks/docs +[tasks_0_code]: tasks.js + +## Running the tests + +1. Set the **GCLOUD_PROJECT** and **GOOGLE_APPLICATION_CREDENTIALS** environment variables. + +1. Run the tests: + + With **npm**: + + npm test + + With **yarn**: + + yarn test diff --git a/cloudtasks/package.json b/cloudtasks/package.json new file mode 100644 index 0000000000..ffdbca3635 --- /dev/null +++ b/cloudtasks/package.json @@ -0,0 +1,36 @@ +{ + "name": "nodejs-docs-samples-cloudtasks", + "version": "0.0.0", + "license": "Apache-2.0", + "author": "Google Inc.", + "private": true, + "repository": "GoogleCloudPlatform/nodejs-docs-samples", + "engines": { + "node": ">=4.0.0" + }, + "scripts": { + "lint": "repo-tools lint", + "pretest": "npm run lint", + "system-test": "ava -T 3m --verbose --fail-fast system-test/*.test.js", + "test": "repo-tools test run --cmd npm -- run system-test" + }, + "dependencies": { + "googleapis": "22.2.0", + "yargs": "10.0.3" + }, + "devDependencies": { + "@google-cloud/nodejs-repo-tools": "2.1.0", + "ava": "0.23.0" + }, + "cloud-repo-tools": { + "requiresKeyFile": true, + "requiresProjectId": true, + "test": { + "build": { + "requiredEnvVars": [ + "GCP_QUEUE" + ] + } + } + } +} diff --git a/cloudtasks/system-test/tasks.test.js b/cloudtasks/system-test/tasks.test.js new file mode 100644 index 0000000000..471f539c55 --- /dev/null +++ b/cloudtasks/system-test/tasks.test.js @@ -0,0 +1,73 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const path = require(`path`); +const test = require(`ava`); +const tools = require(`@google-cloud/nodejs-repo-tools`); + +const { + spawnAsyncWithIO, + runAsync +} = require(`@google-cloud/nodejs-repo-tools`); + +const PROJECT_ID = process.env.GCLOUD_PROJECT; +const QUEUE = process.env.GCP_QUEUE; +const cmd = `node tasks.js`; +const cwd = path.join(__dirname, `..`); + +test.before((t) => { + if (!QUEUE) { + t.fail('You must set the GCP_QUEUE environment variable!'); + } +}); +test.before(tools.checkCredentials); + +let task; + +test.serial(`should create a task`, async (t) => { + const args = [`create`, PROJECT_ID, `us-central1`, QUEUE]; + const results = await spawnAsyncWithIO(cmd, args, cwd); + t.regex(results.output, /Created task/); +}); + +test.serial(`should pull a task`, async (t) => { + t.plan(0); + await tools.tryTest(async (assert) => { + const args = [`pull`, PROJECT_ID, `us-central1`, QUEUE]; + const results = await spawnAsyncWithIO(cmd, args, cwd); + const matches = results.output.match(/^Pulled task ({.+})$/); + if (matches) { + const json = JSON.parse(matches[1]); + if (json.tasks && json.tasks.length) { + task = JSON.stringify(json.tasks[0]); + } + } else { + throw new Error(`Should have pulled task.\n${results.output}`); + } + assert(results.output.includes(`Pulled task`)); + }).start(); +}); + +test.serial(`should acknowledge a task`, async (t) => { + if (task) { + const command = `${cmd} acknowledge '${task}'`; + const output = await runAsync(command, cwd); + t.regex(output, /Acknowledged task/); + } else { + t.fail('no task to acknowledge'); + } +}); diff --git a/cloudtasks/tasks.js b/cloudtasks/tasks.js new file mode 100644 index 0000000000..a39218c14a --- /dev/null +++ b/cloudtasks/tasks.js @@ -0,0 +1,198 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +function createTask (project, location, queue) { + // [START cloud_tasks_create_task] + const google = require('googleapis'); + const cloudtasks = google.cloudtasks('v2beta2'); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const project = 'Project ID, e.g. my-project-id'; + // const location = 'Location of queue, e.g. us-central1'; + // const queue = 'Queue ID, e.g. queue-1'; + + authorize((authClient) => { + const task = { + pull_message: { + payload: Buffer.from('a message for the recipient').toString('base64') + } + }; + + const request = { + parent: `projects/${project}/locations/${location}/queues/${queue}`, + resource: { + task: task + }, + auth: authClient + }; + + cloudtasks.projects.locations.queues.tasks.create(request, (err, response) => { + if (err) { + console.error('ERROR:', err); + return; + } + + console.log(`Created task ${response.name}.`); + console.log(JSON.stringify(response, null, 2)); + }); + }); + + function authorize (callback) { + google.auth.getApplicationDefault(function (err, authClient) { + if (err) { + console.error('authentication failed: ', err); + return; + } + if (authClient.createScopedRequired && authClient.createScopedRequired()) { + var scopes = ['https://www.googleapis.com/auth/cloud-platform']; + authClient = authClient.createScoped(scopes); + } + callback(authClient); + }); + } + // [END cloud_tasks_create_task] +} + +function pullTask (project, location, queue) { + // [START cloud_tasks_pull_task] + const google = require('googleapis'); + const cloudtasks = google.cloudtasks('v2beta2'); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const project = 'Project ID, e.g. my-project-id'; + // const location = 'Location of queue, e.g. us-central1'; + // const queue = 'Queue ID, e.g. queue-1'; + + authorize((authClient) => { + const pullOptions = { + maxTasks: 1, + leaseDuration: '600s', + responseView: 'FULL' + }; + + const request = { + name: `projects/${project}/locations/${location}/queues/${queue}`, + resource: pullOptions, + auth: authClient + }; + + cloudtasks.projects.locations.queues.tasks.pull(request, (err, response) => { + if (err) { + console.error(err); + return; + } + + console.log('Pulled task %j', response); + }); + }); + + function authorize (callback) { + google.auth.getApplicationDefault(function (err, authClient) { + if (err) { + console.error('authentication failed: ', err); + return; + } + if (authClient.createScopedRequired && authClient.createScopedRequired()) { + var scopes = ['https://www.googleapis.com/auth/cloud-platform']; + authClient = authClient.createScoped(scopes); + } + callback(authClient); + }); + } + // [END cloud_tasks_pull_task] +} + +function acknowledgeTask (task) { + // [START cloud_tasks_acknowledge_task] + const google = require('googleapis'); + const cloudtasks = google.cloudtasks('v2beta2'); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const task = { + // name: 'projects/YOUR_PROJECT_ID/locations/us-central1/queues/YOUR_QUEUE_ID/tasks/YOUR_TASK_ID, + // scheduleTime: '2017-11-01T21:02:28.994Z' + // }; + + authorize((authClient) => { + const request = { + name: task.name, + resource: { + scheduleTime: task.scheduleTime + }, + auth: authClient + }; + + cloudtasks.projects.locations.queues.tasks.acknowledge(request, (err, response) => { + if (err) { + console.error(err); + return; + } + + console.log(`Acknowledged task ${task.name}.`); + }); + }); + + function authorize (callback) { + google.auth.getApplicationDefault(function (err, authClient) { + if (err) { + console.error('authentication failed: ', err); + return; + } + if (authClient.createScopedRequired && authClient.createScopedRequired()) { + var scopes = ['https://www.googleapis.com/auth/cloud-platform']; + authClient = authClient.createScoped(scopes); + } + callback(authClient); + }); + } + // [END cloud_tasks_acknowledge_task] +} + +require(`yargs`) // eslint-disable-line + .demand(1) + .command( + `create `, + `Create a task.`, + {}, + (opts) => createTask(opts.project, opts.location, opts.queue) + ) + .command( + `pull `, + `Pull a task.`, + {}, + (opts) => pullTask(opts.project, opts.location, opts.queue) + ) + .command( + `acknowledge `, + `Acknowledge a task.`, + {}, + (opts) => acknowledgeTask(JSON.parse(opts.task)) + ) + .example(`node $0 create my-project-id us-central1 my-queue`) + .example(`node $0 pull my-project-id us-central1 my-queue`) + .example(`node $0 acknowledge '{"name":"projects/my-project-id/locations/us-central1/queues/my-queue/tasks/1234","scheduleTime":"2017-11-01T22:27:53.628279Z"}'`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/cloud-tasks/docs`) + .help() + .strict().argv;