diff --git a/bigquery/package.json b/bigquery/package.json index 6619a6ea77..15025ea215 100644 --- a/bigquery/package.json +++ b/bigquery/package.json @@ -9,6 +9,7 @@ "system-test": "mocha -R spec -t 120000 --require intelli-espower-loader ../system-test/_setup.js system-test/*.test.js" }, "dependencies": { + "@google-cloud/bigquery": "^0.1.1", "async": "^1.5.2", "gcloud": "^0.37.0", "request": "^2.72.0" diff --git a/bigquery/query.js b/bigquery/query.js new file mode 100644 index 0000000000..ceabb5bda5 --- /dev/null +++ b/bigquery/query.js @@ -0,0 +1,177 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START complete] +/** + * Command-line application to perform an synchronous query in BigQuery. + * + * This sample is used on this page: + * + * https://cloud.google.com/bigquery/querying-data + * + * For more information, see the README.md under /bigquery. + */ + +'use strict'; + +// [START auth] +// By default, gcloud will authenticate using the service account file specified +// by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use the +// project specified by the GCLOUD_PROJECT environment variable. See +// https://googlecloudplatform.github.io/gcloud-node/#/docs/guides/authentication +var BigQuery = require('@google-cloud/bigquery'); + +// Instantiate the bigquery client +var bigquery = BigQuery(); +// [END auth] + +// [START sync_query] +/** + * Run a synchronous query. + * @param {string} query The BigQuery query to run, as a string. + * @param {function} callback Callback function to receive query results. + */ +function syncQuery (query, callback) { + if (!query) { + return callback(new Error('"query" is required!')); + } + + // Construct query object. + // Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query + var queryObj = { + query: query, + timeoutMs: 10000 // Time out after 10 seconds. + }; + + // Run query + bigquery.query(queryObj, function (err, rows) { + if (err) { + return callback(err); + } + + console.log('SyncQuery: found %d rows!', rows.length); + return callback(null, rows); + }); +} +// [END sync_query] + +// [START async_query] +/** + * Run an asynchronous query. + * @param {string} query The BigQuery query to run, as a string. + * @param {function} callback Callback function to receive job data. + */ +function asyncQuery (query, callback) { + if (!query) { + return callback(new Error('"query" is required!')); + } + + // Construct query object + // Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query + var queryObj = { + query: query + }; + + // Submit query asynchronously + bigquery.startQuery(queryObj, function (err, job) { + if (err) { + return callback(err); + } + + console.log('AsyncQuery: submitted job %s!', job.id); + return callback(null, job); + }); +} + +/** + * Poll an asynchronous query job for results. + * @param {object} jobId The ID of the BigQuery job to poll. + * @param {function} callback Callback function to receive query results. + */ +function asyncPoll (jobId, callback) { + if (!jobId) { + return callback(new Error('"jobId" is required!')); + } + + // Check for job status + var job = bigquery.job(jobId); + job.getMetadata(function (err, metadata) { + if (err) { + return callback(err); + } + console.log('Job status: %s', metadata.status.state); + + // If job is done, get query results; if not, return an error. + if (metadata.status.state === 'DONE') { + job.getQueryResults(function (err, rows) { + if (err) { + return callback(err); + } + + console.log('AsyncQuery: polled job %s; got %d rows!', jobId, rows.length); + return callback(null, rows); + }); + } else { + return callback(new Error('Job %s is not done', jobId)); + } + }); +} +// [END async_query] + +// [START usage] +function printUsage () { + console.log('Usage:'); + console.log('\nCommands:\n'); + console.log('\tnode query sync QUERY'); + console.log('\tnode query async QUERY'); + console.log('\tnode query poll JOB_ID'); + console.log('\nExamples:\n'); + console.log('\tnode query sync "SELECT * FROM publicdata:samples.natality LIMIT 5;"'); + console.log('\tnode query async "SELECT * FROM publicdata:samples.natality LIMIT 5;"'); + console.log('\tnode query poll 12345'); +} +// [END usage] + +// The command-line program: +var program = { + // Print usage instructions. + printUsage: printUsage, + + // Exports + asyncQuery: asyncQuery, + asyncPoll: asyncPoll, + syncQuery: syncQuery, + bigquery: bigquery, + + // Run the sample. + main: function (args, cb) { + var command = args.shift(); + var arg = args.shift(); + if (command === 'sync') { + this.syncQuery(arg, cb); + } else if (command === 'async') { + this.asyncQuery(arg, cb); + } else if (command === 'poll') { + this.asyncPoll(arg, cb); + } else { + this.printUsage(); + } + } +}; + +if (module === require.main) { + program.main(process.argv.slice(2), console.log); +} +// [END complete] + +module.exports = program; diff --git a/bigquery/system-test/query.test.js b/bigquery/system-test/query.test.js new file mode 100644 index 0000000000..a680565e4c --- /dev/null +++ b/bigquery/system-test/query.test.js @@ -0,0 +1,59 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +var example = require('../query'); + +describe('bigquery:query', function () { + describe('sync_query', function () { + it('should fetch data given a query', function (done) { + example.syncQuery('SELECT * FROM publicdata:samples.natality LIMIT 5;', + function (err, data) { + assert.ifError(err); + assert.notEqual(data, null); + assert(Array.isArray(data)); + assert(data.length === 5); + done(); + } + ); + }); + }); + + describe('async_query', function () { + it('should submit a job and fetch its results', function (done) { + example.asyncQuery('SELECT * FROM publicdata:samples.natality LIMIT 5;', + function (err, job) { + assert.ifError(err); + assert.notEqual(job.id, null); + + var poller = function (tries) { + example.asyncPoll(job.id, function (err, data) { + if (!err || tries === 0) { + assert.ifError(err); + assert.notEqual(data, null); + assert(Array.isArray(data)); + assert(data.length === 5); + done(); + } else { + setTimeout(function () { poller(tries - 1); }, 1000); + } + }); + }; + + poller(5); + } + ); + }); + }); +}); diff --git a/bigquery/test/query.test.js b/bigquery/test/query.test.js new file mode 100644 index 0000000000..7f801d35ed --- /dev/null +++ b/bigquery/test/query.test.js @@ -0,0 +1,264 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +var proxyquire = require('proxyquire').noCallThru(); + +function getSample () { + var natalityMock = [ + { year: '2001' }, + { year: '2002' }, + { year: '2003' }, + { year: '2004' }, + { year: '2005' } + ]; + + var metadataMock = { status: { state: 'DONE' } }; + + var jobId = 'abc'; + + var jobMock = { + id: jobId, + getQueryResults: sinon.stub().callsArgWith(0, null, natalityMock), + getMetadata: sinon.stub().callsArgWith(0, null, metadataMock) + }; + + var bigqueryMock = { + job: sinon.stub().returns(jobMock), + startQuery: sinon.stub().callsArgWith(1, null, jobMock), + query: sinon.stub().callsArgWith(1, null, natalityMock) + }; + + var BigQueryMock = sinon.stub().returns(bigqueryMock); + + return { + program: proxyquire('../query', { + '@google-cloud/bigquery': BigQueryMock + }), + mocks: { + BigQuery: BigQueryMock, + bigquery: bigqueryMock, + natality: natalityMock, + metadata: metadataMock, + job: jobMock + }, + jobId: jobId + }; +} + +describe('bigquery:query', function () { + describe('main', function () { + it('should show usage based on arguments', function () { + var program = getSample().program; + sinon.stub(program, 'printUsage'); + + program.main([]); + assert(program.printUsage.calledOnce); + + program.main(['-h']); + assert(program.printUsage.calledTwice); + + program.main(['--help']); + assert(program.printUsage.calledThrice); + }); + + it('should run the correct commands', function () { + var program = getSample().program; + sinon.stub(program, 'syncQuery'); + sinon.stub(program, 'asyncQuery'); + sinon.stub(program, 'asyncPoll'); + + program.main(['sync']); + assert(program.syncQuery.calledOnce); + + program.main(['async']); + assert(program.asyncQuery.calledOnce); + + program.main(['poll']); + assert(program.asyncPoll.calledOnce); + }); + + it('should execute queries', function () { + var example = getSample(); + sinon.stub(example.program, 'syncQuery'); + + example.program.main(['foo'], function (err, data) { + assert.ifError(err); + assert(example.program.syncQuery.calledWith({ query: 'foo' })); + assert.deepEqual(data, example.mocks.natality); + }); + }); + }); + + describe('syncQuery', function () { + var query = 'foo'; + + it('should return results', function () { + var example = getSample(); + example.program.syncQuery(query, + function (err, data) { + assert.ifError(err); + assert(example.mocks.bigquery.query.called); + assert.deepEqual(data, example.mocks.natality); + assert(console.log.calledWith( + 'SyncQuery: found %d rows!', + data.length + )); + } + ); + }); + + it('should require a query', function () { + var example = getSample(); + example.program.syncQuery(null, function (err, data) { + assert.deepEqual(err, Error('"query" is required!')); + assert.equal(data, undefined); + }); + }); + + it('should handle error', function () { + var error = Error('syncQueryError'); + var example = getSample(); + example.mocks.bigquery.query = sinon.stub().callsArgWith(1, error); + example.program.syncQuery(query, function (err, data) { + assert.deepEqual(err, error); + assert.equal(data, undefined); + }); + }); + }); + + describe('asyncQuery', function () { + var query = 'foo'; + + it('should submit a job', function () { + var example = getSample(); + example.program.asyncQuery(query, + function (err, job) { + assert.ifError(err); + assert(example.mocks.bigquery.startQuery.called); + assert.deepEqual(example.mocks.job, job); + assert(console.log.calledWith( + 'AsyncQuery: submitted job %s!', example.jobId + )); + } + ); + }); + + it('should require a query', function () { + var example = getSample(); + example.program.asyncQuery(null, function (err, job) { + assert.deepEqual(err, Error('"query" is required!')); + assert.equal(job, undefined); + }); + }); + + it('should handle error', function () { + var error = Error('asyncQueryError'); + var example = getSample(); + example.mocks.bigquery.startQuery = sinon.stub().callsArgWith(1, error); + example.program.asyncQuery(query, function (err, job) { + assert.deepEqual(err, error); + assert.equal(job, undefined); + }); + }); + }); + + describe('asyncPoll', function () { + it('should get the results of a job given its ID', function () { + var example = getSample(); + example.mocks.bigquery.job = sinon.stub().returns(example.mocks.job); + example.program.asyncPoll(example.jobId, + function (err, rows) { + assert.ifError(err); + assert(example.mocks.job.getQueryResults.called); + assert(console.log.calledWith( + 'AsyncQuery: polled job %s; got %d rows!', + example.jobId, + example.mocks.natality.length + )); + } + ); + }); + + it('should report the status of a job', function () { + var example = getSample(); + example.program.asyncPoll(example.jobId, function (err, rows) { + assert.ifError(err); + assert(example.mocks.job.getMetadata.called); + assert(console.log.calledWith( + 'Job status: %s', + example.mocks.metadata.status.state + )); + }); + }); + + it('should check whether a job is finished', function () { + var example = getSample(); + + var pendingState = { status: { state: 'PENDING' } }; + example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, pendingState); + example.program.asyncPoll(example.jobId, function (err, rows) { + assert.deepEqual(err, Error('Job %s is not done', example.jobId)); + assert(console.log.calledWith('Job status: %s', pendingState.status.state)); + assert(example.mocks.job.getMetadata.called); + assert.equal(example.mocks.job.getQueryResults.called, false); + assert.equal(rows, undefined); + }); + + var doneState = { status: { state: 'DONE' } }; + example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, doneState); + example.program.asyncPoll(example.jobId, function (err, rows) { + assert.ifError(err); + assert(console.log.calledWith('Job status: %s', doneState.status.state)); + assert(example.mocks.job.getMetadata.called); + assert(example.mocks.job.getQueryResults.called); + assert.equal(rows, example.mocks.natality); + }); + }); + + it('should require a job ID', function () { + var example = getSample(); + example.program.asyncPoll(null, function (err, rows) { + assert.deepEqual(err, Error('"jobId" is required!')); + assert.equal(rows, undefined); + }); + }); + + it('should handle error', function () { + var error = Error('asyncPollError'); + var example = getSample(); + example.mocks.job.getQueryResults = sinon.stub().callsArgWith(0, error); + example.program.asyncPoll(example.jobId, function (err, rows) { + assert.deepEqual(err, error); + assert.equal(rows, undefined); + }); + }); + }); + + describe('printUsage', function () { + it('should print usage', function () { + var program = getSample().program; + program.printUsage(); + assert(console.log.calledWith('Usage:')); + assert(console.log.calledWith('\nCommands:\n')); + assert(console.log.calledWith('\tnode query sync QUERY')); + assert(console.log.calledWith('\tnode query async QUERY')); + assert(console.log.calledWith('\tnode query poll JOB_ID')); + assert(console.log.calledWith('\nExamples:\n')); + assert(console.log.calledWith('\tnode query sync "SELECT * FROM publicdata:samples.natality LIMIT 5;"')); + assert(console.log.calledWith('\tnode query async "SELECT * FROM publicdata:samples.natality LIMIT 5;"')); + assert(console.log.calledWith('\tnode query poll 12345')); + }); + }); +});