Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor BigQuery samples. #297

Merged
merged 1 commit into from
Aug 17, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions bigquery/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
<version>${project.oauth.version}</version>
<version>1.21.0</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-jackson2</artifactId>
<version>${project.http.version}</version>
<version>1.21.0</version>
</dependency>
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client-jetty</artifactId>
<version>${project.oauth.version}</version>
<version>1.21.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
Expand All @@ -48,6 +48,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -59,8 +60,6 @@
</dependencies>

<properties>
<project.http.version>1.21.0</project.http.version>
<project.oauth.version>1.21.0</project.oauth.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
* License for the specific language governing permissions and limitations under
* the License.
*/

package com.google.cloud.bigquery.samples;


import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults;
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
Expand All @@ -28,11 +26,10 @@
import java.util.Iterator;
import java.util.Scanner;


/**
* Example of authorizing with BigQuery and reading from a public dataset.
*/
public class AsyncQuerySample extends BigqueryUtils {
public class AsyncQuerySample {
// [START main]
/**
* Prompts for all the parameters required to make a query.
Expand All @@ -41,23 +38,20 @@ public class AsyncQuerySample extends BigqueryUtils {
* @throws IOException IOException
* @throws InterruptedException InterruptedException
*/
public static void main(final String[] args)
throws IOException, InterruptedException {
public static void main(final String[] args) throws IOException, InterruptedException {
Scanner scanner = new Scanner(System.in);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some reason these aren't command line args, with defaults?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was the way it was in the existing sample, and it didn't bother me as much as some of the other things I fixed. I can look at changing this, too.

System.out.println("Enter your project id: ");
String projectId = scanner.nextLine();
System.out.println("Enter your query string: ");
String queryString = scanner.nextLine();
System.out.println("Run query in batch mode? [true|false] ");
boolean batch = Boolean.valueOf(scanner.nextLine());
System.out.println("Enter how often to check if your job is complete "
+ "(milliseconds): ");
System.out.println("Enter how often to check if your job is complete " + "(milliseconds): ");
long waitTime = scanner.nextLong();
scanner.close();
Iterator<GetQueryResultsResponse> pages = run(projectId, queryString,
batch, waitTime);
Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, batch, waitTime);
while (pages.hasNext()) {
printRows(pages.next().getRows(), System.out);
BigQueryUtils.printRows(pages.next().getRows(), System.out);
}
}
// [END main]
Expand All @@ -70,58 +64,53 @@ public static void main(final String[] args)
* @param queryString Query we want to run against BigQuery
* @param batch True if you want to batch the queries
* @param waitTime How long to wait before retries
* @return An interator to the result of your pages
* @return An iterator to the result of your pages
* @throws IOException Thrown if there's an IOException
* @throws InterruptedException Thrown if there's an Interrupted Exception
*/
public static Iterator<GetQueryResultsResponse> run(final String projectId,
final String queryString,
final boolean batch,
final long waitTime)
public static Iterator<GetQueryResultsResponse> run(
final String projectId, final String queryString, final boolean batch, final long waitTime)
throws IOException, InterruptedException {

Bigquery bigquery = BigqueryServiceFactory.getService();
Bigquery bigquery = BigQueryServiceFactory.getService();

Job query = asyncQuery(bigquery, projectId, queryString, batch);
Bigquery.Jobs.Get getRequest = bigquery.jobs().get(
projectId, query.getJobReference().getJobId());
Bigquery.Jobs.Get getRequest =
bigquery.jobs().get(projectId, query.getJobReference().getJobId());

//Poll every waitTime milliseconds,
//retrying at most retries times if there are errors
pollJob(getRequest, waitTime);
// Poll every waitTime milliseconds,
// retrying at most retries times if there are errors
BigQueryUtils.pollJob(getRequest, waitTime);

GetQueryResults resultsRequest = bigquery.jobs().getQueryResults(
projectId, query.getJobReference().getJobId());
GetQueryResults resultsRequest =
bigquery.jobs().getQueryResults(projectId, query.getJobReference().getJobId());

return getPages(resultsRequest);
return BigQueryUtils.getPages(resultsRequest);
}
// [END run]

// [START asyncQuery]
/**
* Inserts an asynchronous query Job for a particular query.
*
* @param bigquery an authorized BigQuery client
* @param bigquery an authorized BigQuery client
* @param projectId a String containing the project ID
* @param querySql the actual query string
* @param batch True if you want to run the query as BATCH
* @return a reference to the inserted query job
* @throws IOException Thrown if there's a network exception
*/
public static Job asyncQuery(final Bigquery bigquery,
final String projectId,
final String querySql,
final boolean batch) throws IOException {
public static Job asyncQuery(
final Bigquery bigquery, final String projectId, final String querySql, final boolean batch)
throws IOException {

JobConfigurationQuery queryConfig = new JobConfigurationQuery()
.setQuery(querySql);
JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(querySql);

if (batch) {
queryConfig.setPriority("BATCH");
}

Job job = new Job().setConfiguration(
new JobConfiguration().setQuery(queryConfig));
Job job = new Job().setConfiguration(new JobConfiguration().setQuery(queryConfig));

return bigquery.jobs().insert(projectId, job).execute();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,14 @@
import java.util.Collection;

/**
* This class creates our Service to connect to Bigquery including auth.
* This class creates our Service to connect to BigQuery including auth.
*/
public final class BigqueryServiceFactory {
public final class BigQueryServiceFactory {

/**
* Private constructor to disable creation of this utility Factory class.
*/
private BigqueryServiceFactory() {

}
private BigQueryServiceFactory() {}

/**
* Singleton service used through the app.
Expand All @@ -52,9 +50,9 @@ private BigqueryServiceFactory() {
private static Object serviceLock = new Object();

/**
* Threadsafe Factory that provides an authorized Bigquery service.
* @return The Bigquery service
* @throws IOException Thronw if there is an error connecting to Bigquery.
* Threadsafe Factory that provides an authorized BigQuery service.
* @return The BigQuery service
* @throws IOException Throw if there is an error connecting to BigQuery.
*/
public static Bigquery getService() throws IOException {
if (service == null) {
Expand All @@ -68,7 +66,7 @@ public static Bigquery getService() throws IOException {
}

/**
* Creates an authorized client to Google Bigquery.
* Creates an authorized client to Google BigQuery.
*
* @return The BigQuery Service
* @throws IOException Thrown if there is an error connecting
Expand All @@ -78,18 +76,19 @@ private static Bigquery createAuthorizedClient() throws IOException {
// Create the credential
HttpTransport transport = new NetHttpTransport();
JsonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);

// Depending on the environment that provides the default credentials (e.g. Compute Engine, App
// Engine), the credentials may require us to specify the scopes we need explicitly.
// Check for this case, and inject the Bigquery scope if required.
// Check for this case, and inject the BigQuery scope if required.
if (credential.createScopedRequired()) {
Collection<String> bigqueryScopes = BigqueryScopes.all();
credential = credential.createScoped(bigqueryScopes);
}

return new Bigquery.Builder(transport, jsonFactory, credential)
.setApplicationName("BigQuery Samples").build();
.setApplicationName("BigQuery Samples")
.build();
}
// [END get_service]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,13 @@
/**
* Helper functions for the other classes.
*/
public class BigqueryUtils {
public class BigQueryUtils {

/**
* Private contructor to prevent creation of this class, which is just all
* Private constructor to prevent creation of this class, which is just all
* static helper methods.
*/
protected BigqueryUtils() {
}
private BigQueryUtils() {}

/**
* Print rows to the output stream in a formatted way.
Expand Down Expand Up @@ -76,9 +75,8 @@ public static Job pollJob(final Bigquery.Jobs.Get request, final long interval)
throws IOException, InterruptedException {
Job job = request.execute();
while (!job.getStatus().getState().equals("DONE")) {
System.out.println("Job is "
+ job.getStatus().getState()
+ " waiting " + interval + " milliseconds...");
System.out.println(
"Job is " + job.getStatus().getState() + " waiting " + interval + " milliseconds...");
Thread.sleep(interval);
job = request.execute();
}
Expand Down Expand Up @@ -165,9 +163,10 @@ public void remove() {
public static TableSchema loadSchema(final Reader schemaSource) {
TableSchema sourceSchema = new TableSchema();

List<TableFieldSchema> fields = (new Gson())
.<List<TableFieldSchema>>fromJson(schemaSource,
(new ArrayList<TableFieldSchema>()).getClass());
List<TableFieldSchema> fields =
(new Gson())
.<List<TableFieldSchema>>fromJson(
schemaSource, (new ArrayList<TableFieldSchema>()).getClass());

sourceSchema.setFields(fields);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
/**
* Sample of how to Export Cloud Data.
*/
public class ExportDataCloudStorageSample {
public class ExportDataCloudStorageSample {
/**
* Protected constructor since this is a collection of static functions.
*/
Expand All @@ -42,20 +42,17 @@ protected ExportDataCloudStorageSample() {
* @throws InterruptedException Should never be thrown.
*/
// [START main]
public static void main(final String[] args)
throws IOException, InterruptedException {
public static void main(final String[] args) throws IOException, InterruptedException {
Scanner scanner = new Scanner(System.in);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sure there is a good reason why we don't use command line args, w/ reasonable defaults.

System.out.println("Enter your project id: ");
String projectId = scanner.nextLine();
System.out.println("Enter your dataset id: ");
String datasetId = scanner.nextLine();
System.out.println("Enter your table id: ");
String tableId = scanner.nextLine();
System.out.println("Enter the Google Cloud Storage Path to which you'd "
+ "like to export: ");
System.out.println("Enter the Google Cloud Storage Path to which you'd " + "like to export: ");
String cloudStoragePath = scanner.nextLine();
System.out.println("Enter how often to check if your job is complete "
+ "(milliseconds): ");
System.out.println("Enter how often to check if your job is complete " + "(milliseconds): ");
long interval = scanner.nextLong();
scanner.close();

Expand All @@ -79,30 +76,33 @@ public static void run(
final String projectId,
final String datasetId,
final String tableId,
final long interval) throws IOException, InterruptedException {
final long interval)
throws IOException, InterruptedException {

Bigquery bigquery = BigqueryServiceFactory.getService();
Bigquery bigquery = BigQueryServiceFactory.getService();

Job extractJob = extractJob(
bigquery,
cloudStoragePath,
new TableReference()
.setProjectId(projectId)
.setDatasetId(datasetId)
.setTableId(tableId));
Job extractJob =
extractJob(
bigquery,
cloudStoragePath,
new TableReference()
.setProjectId(projectId)
.setDatasetId(datasetId)
.setTableId(tableId));

Bigquery.Jobs.Get getJob = bigquery.jobs().get(
extractJob.getJobReference().getProjectId(),
extractJob.getJobReference().getJobId());
Bigquery.Jobs.Get getJob =
bigquery
.jobs()
.get(
extractJob.getJobReference().getProjectId(),
extractJob.getJobReference().getJobId());

BigqueryUtils.pollJob(getJob, interval);
BigQueryUtils.pollJob(getJob, interval);

System.out.println("Export is Done!");

}
// [END run]


/**
* A job that extracts data from a table.
* @param bigquery Bigquery service to use
Expand All @@ -113,16 +113,17 @@ public static void run(
*/
// [START extract_job]
public static Job extractJob(
final Bigquery bigquery,
final String cloudStoragePath,
final TableReference table) throws IOException {
final Bigquery bigquery, final String cloudStoragePath, final TableReference table)
throws IOException {

JobConfigurationExtract extract = new JobConfigurationExtract()
.setSourceTable(table)
.setDestinationUri(cloudStoragePath);
JobConfigurationExtract extract =
new JobConfigurationExtract().setSourceTable(table).setDestinationUri(cloudStoragePath);

return bigquery.jobs().insert(table.getProjectId(),
new Job().setConfiguration(new JobConfiguration().setExtract(extract)))
return bigquery
.jobs()
.insert(
table.getProjectId(),
new Job().setConfiguration(new JobConfiguration().setExtract(extract)))
.execute();
}
// [END extract_job]
Expand Down
Loading