Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor query samples to follow python canonical and add disable cache sample #215

Merged
merged 6 commits into from
Oct 20, 2018
Merged

Refactor query samples to follow python canonical and add disable cache sample #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f29fef3
refactors to follow python canonical and adds disable cache sample
alixhami Oct 17, 2018
158f112
update disable cache query to match canonical and add locations
alixhami Oct 17, 2018
0b7db94
fix lint
alixhami Oct 17, 2018
01efea2
remove unused vars
alixhami Oct 17, 2018
647d7a1
fix lint again
alixhami Oct 17, 2018
3741653
Merge branch 'master' into bq-no-cache
JustinBeckwith Oct 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 58 additions & 81 deletions samples/queries.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,16 @@

'use strict';

// [START bigquery_simple_app_all]
function printResult(rows) {
// [START bigquery_simple_app_print]
console.log('Query Results:');
rows.forEach(row => {
const url = row['url'];
const viewCount = row['view_count'];
console.log(`url: ${url}, ${viewCount} views`);
});
// [END bigquery_simple_app_print]
}

async function queryStackOverflow(projectId) {
async function queryStackOverflow() {
// [START bigquery_simple_app_all]
// [START bigquery_simple_app_deps]
// Imports the Google Cloud client library
const BigQuery = require('@google-cloud/bigquery');
// [END bigquery_simple_app_deps]

// [START bigquery_simple_app_client]
/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// const projectId = "your-project-id";

// Creates a client
const bigquery = new BigQuery({projectId});
const bigquery = new BigQuery();
// [END bigquery_simple_app_client]

// [START bigquery_simple_app_query]
Expand All @@ -55,116 +39,109 @@ async function queryStackOverflow(projectId) {
ORDER BY view_count DESC
LIMIT 10`;

// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
const options = {
query: sqlQuery,
useLegacySql: false, // Use standard SQL syntax for queries.
// Location must match that of the dataset(s) referenced in the query.
location: 'US',
};

// Runs the query
const [rows] = await bigquery.query(options);
printResult(rows);
// [END bigquery_simple_app_query]

// [START bigquery_simple_app_print]
console.log('Query Results:');
rows.forEach(row => {
const url = row['url'];
const viewCount = row['view_count'];
console.log(`url: ${url}, ${viewCount} views`);
});
// [END bigquery_simple_app_print]
}
// [END bigquery_simple_app_all]

async function syncQuery(sqlQuery, projectId) {
async function query() {
// [START bigquery_query]
// Imports the Google Cloud client library
const BigQuery = require('@google-cloud/bigquery');

/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// const projectId = "your-project-id";
// const sqlQuery = "SELECT * FROM publicdata.samples.natality LIMIT 5;";

// Creates a client
const bigquery = new BigQuery({projectId});
const bigquery = new BigQuery();

// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
const query = `SELECT name

This comment was marked as spam.

Sign in to view

FROM \`bigquery-public-data.usa_names.usa_1910_2013\`
WHERE state = 'TX'
LIMIT 100`;
const options = {
query: sqlQuery,
timeoutMs: 10000, // Time out after 10 seconds.
useLegacySql: false, // Use standard SQL syntax for queries.
query: query,
// Location must match that of the dataset(s) referenced in the query.
location: 'US',
};

// Runs the query
const [rows] = await bigquery.query(options);
// Runs the query as a job
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);

// Waits for the query to finish
const [rows] = await job.getQueryResults();

// Prints the results
console.log('Rows:');
rows.forEach(row => console.log(row));
// [END bigquery_query]
}

async function asyncQuery(sqlQuery, projectId) {
// [START bigquery_query]
async function queryDisableCache() {
// [START bigquery_query_no_cache]
// Imports the Google Cloud client library
const BigQuery = require('@google-cloud/bigquery');

/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// const projectId = "your-project-id";
// const sqlQuery = "SELECT * FROM publicdata.samples.natality LIMIT 5;";

// Creates a client
const bigquery = new BigQuery({projectId});
const bigquery = new BigQuery();

// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
const query = `SELECT corpus
FROM \`bigquery-public-data.samples.shakespeare\`
GROUP BY corpus`;
const options = {
query: sqlQuery,
useLegacySql: false, // Use standard SQL syntax for queries.
query: query,
// Location must match that of the dataset(s) referenced in the query.
location: 'US',
useQueryCache: false,
};

// Runs the query as a job
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);

// Get the job's status
const metadata = await job.getMetadata();

// Check the job's status for errors
const errors = metadata[0].status.errors;
if (errors && errors.length > 0) {
throw errors;
}
console.log(`Job ${job.id} completed.`);

// Waits for the query to finish
const [rows] = await job.getQueryResults();

// Prints the results
console.log('Rows:');
rows.forEach(row => console.log(row));
// [END bigquery_query]
// [END bigquery_query_no_cache]
}

require(`yargs`)
.demand(1)
.command(
`sync <projectId> <sqlQuery>`,
`Run the specified synchronous query.`,
{},
opts => syncQuery(opts.sqlQuery, opts.projectId)
)
.command(
`async <projectId> <sqlQuery>`,
`Start the specified asynchronous query.`,
`stackoverflow`,
`Queries a public Stack Overflow dataset.`,
{},
opts => asyncQuery(opts.sqlQuery, opts.projectId)
queryStackOverflow
)
.command(`query`, `Queries the US Names dataset.`, {}, query)
.command(
`stackoverflow <projectId>`,
`Queries a public Stack Overflow dataset.`,
`disable-cache`,
`Queries the Shakespeare dataset with the cache disabled.`,
{},
opts => queryStackOverflow(opts.projectId)
)
.example(
`node $0 sync my-project-id "SELECT * FROM publicdata.samples.natality LIMIT 5;"`,
`Synchronously queries the natality dataset.`
)
.example(
`node $0 async my-project-id "SELECT * FROM publicdata.samples.natality LIMIT 5;"`,
`Queries the natality dataset as a job.`
queryDisableCache
)
.example(`node $0 stackoverflow`, `Queries a public Stackoverflow dataset.`)
.example(`node $0 query`, `Queries the US Names dataset.`)
.example(
`node $0 shakespeare my-project-id`,
`Queries a public Shakespeare dataset.`
`node $0 disable-cache`,
`Queries the Shakespeare dataset with the cache disabled.`
)
.wrap(120)
.recommendCommands()
Expand Down
38 changes: 7 additions & 31 deletions samples/system-test/queries.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,47 +21,23 @@ const tools = require(`@google-cloud/nodejs-repo-tools`);

const cwd = path.join(__dirname, `..`);
const cmd = `node queries.js`;
const projectId = process.env.GCLOUD_PROJECT;

const sqlQuery = `SELECT * FROM publicdata.samples.natality LIMIT 5;`;
const badQuery = `SELECT * FROM INVALID`;

test.before(tools.checkCredentials);

test(`should query stackoverflow`, async t => {
const output = await tools.runAsync(`${cmd} stackoverflow ${projectId}`, cwd);
const output = await tools.runAsync(`${cmd} stackoverflow`, cwd);
t.true(output.includes(`Query Results:`));
t.true(output.includes(`views`));
});

test(`should run a sync query`, async t => {
const output = await tools.runAsync(
`${cmd} sync ${projectId} "${sqlQuery}"`,
cwd
);
test(`should run a query`, async t => {
const output = await tools.runAsync(`${cmd} query`, cwd);
t.true(output.includes(`Rows:`));
t.true(output.includes(`source_year`));
t.true(output.includes(`name`));
});

test(`should run an async query`, async t => {
const output = await tools.runAsync(
`${cmd} async ${projectId} "${sqlQuery}"`,
cwd
);
test(`should run a query with the cache disabled`, async t => {
const output = await tools.runAsync(`${cmd} disable-cache`, cwd);
t.true(output.includes(`Rows:`));
t.true(output.includes(`source_year`));
});

test.skip(`should handle sync query errors`, async t => {
await t.throws(
tools.runAsync(`${cmd} sync ${projectId} "${badQuery}"`, cwd),
/ERROR:/
);
});

test.skip(`should handle async query errors`, async t => {
await t.throws(
tools.runAsync(`${cmd} async ${projectId} "${badQuery}"`, cwd),
/ERROR:/
);
t.true(output.includes(`corpus`));
});