From a9451b59da694512e79a49bba685c2928835cc2a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Aug 2016 10:40:58 -0700 Subject: [PATCH] Refactor BigQuery samples. Fixes the following issues: - Subclassing a static utility class unncecessarily. - Parsing constants from a JSON file for every test instead of defining actual constants. - Rename Bigquery to BigQuery. - Formatted using google-java-format https://github.com/google/google-java-format --- bigquery/pom.xml | 9 +-- .../bigquery/samples/AsyncQuerySample.java | 57 ++++++------- ...ctory.java => BigQueryServiceFactory.java} | 23 +++--- ...{BigqueryUtils.java => BigQueryUtils.java} | 19 +++-- .../samples/ExportDataCloudStorageSample.java | 61 +++++++------- .../bigquery/samples/GettingStarted.java | 28 ++++--- .../samples/ListDatasetsProjects.java | 21 +++-- .../bigquery/samples/LoadDataCsvSample.java | 71 ++++++++-------- .../bigquery/samples/StreamingSample.java | 57 ++++++------- .../bigquery/samples/SyncQuerySample.java | 49 +++++------ bigquery/src/main/resources/constants.json | 9 --- .../samples/test/AsyncQuerySampleTest.java | 16 ++-- .../samples/test/BigquerySampleTest.java | 81 ------------------- .../bigquery/samples/test/Constants.java | 28 +++++++ .../ExportDataCloudStorageSampleTest.java | 21 ++--- .../samples/test/GettingStartedTest.java | 13 ++- .../test/ListDatasetsProjectsTest.java | 12 ++- .../samples/test/LoadDataCsvSampleTest.java | 23 ++---- .../samples/test/StreamingSampleTest.java | 19 ++--- .../samples/test/SyncQuerySampleTest.java | 18 ++--- 20 files changed, 261 insertions(+), 374 deletions(-) rename bigquery/src/main/java/com/google/cloud/bigquery/samples/{BigqueryServiceFactory.java => BigQueryServiceFactory.java} (80%) rename bigquery/src/main/java/com/google/cloud/bigquery/samples/{BigqueryUtils.java => BigQueryUtils.java} (93%) delete mode 100644 bigquery/src/main/resources/constants.json delete mode 100644 bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java create mode 100644 bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java diff --git a/bigquery/pom.xml b/bigquery/pom.xml index 1859b420b4b..c5ef6e6448f 100644 --- a/bigquery/pom.xml +++ b/bigquery/pom.xml @@ -28,17 +28,17 @@ com.google.oauth-client google-oauth-client - ${project.oauth.version} + 1.21.0 com.google.http-client google-http-client-jackson2 - ${project.http.version} + 1.21.0 com.google.oauth-client google-oauth-client-jetty - ${project.oauth.version} + 1.21.0 com.google.code.gson @@ -48,6 +48,7 @@ junit junit + 4.12 test @@ -59,8 +60,6 @@ - 1.21.0 - 1.21.0 UTF-8 diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java index a76b9e03c51..e44aa537470 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java @@ -13,10 +13,8 @@ * License for the specific language governing permissions and limitations under * the License. */ - package com.google.cloud.bigquery.samples; - import com.google.api.services.bigquery.Bigquery; import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; import com.google.api.services.bigquery.model.GetQueryResultsResponse; @@ -28,11 +26,10 @@ import java.util.Iterator; import java.util.Scanner; - /** * Example of authorizing with BigQuery and reading from a public dataset. */ -public class AsyncQuerySample extends BigqueryUtils { +public class AsyncQuerySample { // [START main] /** * Prompts for all the parameters required to make a query. @@ -41,8 +38,7 @@ public class AsyncQuerySample extends BigqueryUtils { * @throws IOException IOException * @throws InterruptedException InterruptedException */ - public static void main(final String[] args) - throws IOException, InterruptedException { + public static void main(final String[] args) throws IOException, InterruptedException { Scanner scanner = new Scanner(System.in); System.out.println("Enter your project id: "); String projectId = scanner.nextLine(); @@ -50,14 +46,12 @@ public static void main(final String[] args) String queryString = scanner.nextLine(); System.out.println("Run query in batch mode? [true|false] "); boolean batch = Boolean.valueOf(scanner.nextLine()); - System.out.println("Enter how often to check if your job is complete " - + "(milliseconds): "); + System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); long waitTime = scanner.nextLong(); scanner.close(); - Iterator pages = run(projectId, queryString, - batch, waitTime); + Iterator pages = run(projectId, queryString, batch, waitTime); while (pages.hasNext()) { - printRows(pages.next().getRows(), System.out); + BigQueryUtils.printRows(pages.next().getRows(), System.out); } } // [END main] @@ -70,30 +64,28 @@ public static void main(final String[] args) * @param queryString Query we want to run against BigQuery * @param batch True if you want to batch the queries * @param waitTime How long to wait before retries - * @return An interator to the result of your pages + * @return An iterator to the result of your pages * @throws IOException Thrown if there's an IOException * @throws InterruptedException Thrown if there's an Interrupted Exception */ - public static Iterator run(final String projectId, - final String queryString, - final boolean batch, - final long waitTime) + public static Iterator run( + final String projectId, final String queryString, final boolean batch, final long waitTime) throws IOException, InterruptedException { - Bigquery bigquery = BigqueryServiceFactory.getService(); + Bigquery bigquery = BigQueryServiceFactory.getService(); Job query = asyncQuery(bigquery, projectId, queryString, batch); - Bigquery.Jobs.Get getRequest = bigquery.jobs().get( - projectId, query.getJobReference().getJobId()); + Bigquery.Jobs.Get getRequest = + bigquery.jobs().get(projectId, query.getJobReference().getJobId()); - //Poll every waitTime milliseconds, - //retrying at most retries times if there are errors - pollJob(getRequest, waitTime); + // Poll every waitTime milliseconds, + // retrying at most retries times if there are errors + BigQueryUtils.pollJob(getRequest, waitTime); - GetQueryResults resultsRequest = bigquery.jobs().getQueryResults( - projectId, query.getJobReference().getJobId()); + GetQueryResults resultsRequest = + bigquery.jobs().getQueryResults(projectId, query.getJobReference().getJobId()); - return getPages(resultsRequest); + return BigQueryUtils.getPages(resultsRequest); } // [END run] @@ -101,27 +93,24 @@ public static Iterator run(final String projectId, /** * Inserts an asynchronous query Job for a particular query. * - * @param bigquery an authorized BigQuery client + * @param bigquery an authorized BigQuery client * @param projectId a String containing the project ID * @param querySql the actual query string * @param batch True if you want to run the query as BATCH * @return a reference to the inserted query job * @throws IOException Thrown if there's a network exception */ - public static Job asyncQuery(final Bigquery bigquery, - final String projectId, - final String querySql, - final boolean batch) throws IOException { + public static Job asyncQuery( + final Bigquery bigquery, final String projectId, final String querySql, final boolean batch) + throws IOException { - JobConfigurationQuery queryConfig = new JobConfigurationQuery() - .setQuery(querySql); + JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(querySql); if (batch) { queryConfig.setPriority("BATCH"); } - Job job = new Job().setConfiguration( - new JobConfiguration().setQuery(queryConfig)); + Job job = new Job().setConfiguration(new JobConfiguration().setQuery(queryConfig)); return bigquery.jobs().insert(projectId, job).execute(); } diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java similarity index 80% rename from bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java rename to bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java index a9037ef4bd5..9d6daee7185 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java @@ -30,16 +30,14 @@ import java.util.Collection; /** - * This class creates our Service to connect to Bigquery including auth. + * This class creates our Service to connect to BigQuery including auth. */ -public final class BigqueryServiceFactory { +public final class BigQueryServiceFactory { /** * Private constructor to disable creation of this utility Factory class. */ - private BigqueryServiceFactory() { - - } + private BigQueryServiceFactory() {} /** * Singleton service used through the app. @@ -52,9 +50,9 @@ private BigqueryServiceFactory() { private static Object serviceLock = new Object(); /** - * Threadsafe Factory that provides an authorized Bigquery service. - * @return The Bigquery service - * @throws IOException Thronw if there is an error connecting to Bigquery. + * Threadsafe Factory that provides an authorized BigQuery service. + * @return The BigQuery service + * @throws IOException Throw if there is an error connecting to BigQuery. */ public static Bigquery getService() throws IOException { if (service == null) { @@ -68,7 +66,7 @@ public static Bigquery getService() throws IOException { } /** - * Creates an authorized client to Google Bigquery. + * Creates an authorized client to Google BigQuery. * * @return The BigQuery Service * @throws IOException Thrown if there is an error connecting @@ -78,18 +76,19 @@ private static Bigquery createAuthorizedClient() throws IOException { // Create the credential HttpTransport transport = new NetHttpTransport(); JsonFactory jsonFactory = new JacksonFactory(); - GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); // Depending on the environment that provides the default credentials (e.g. Compute Engine, App // Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the Bigquery scope if required. + // Check for this case, and inject the BigQuery scope if required. if (credential.createScopedRequired()) { Collection bigqueryScopes = BigqueryScopes.all(); credential = credential.createScoped(bigqueryScopes); } return new Bigquery.Builder(transport, jsonFactory, credential) - .setApplicationName("BigQuery Samples").build(); + .setApplicationName("BigQuery Samples") + .build(); } // [END get_service] diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java similarity index 93% rename from bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java rename to bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java index 743c29eac79..33f9fb247d6 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java @@ -38,14 +38,13 @@ /** * Helper functions for the other classes. */ -public class BigqueryUtils { +public class BigQueryUtils { /** - * Private contructor to prevent creation of this class, which is just all + * Private constructor to prevent creation of this class, which is just all * static helper methods. */ - protected BigqueryUtils() { - } + private BigQueryUtils() {} /** * Print rows to the output stream in a formatted way. @@ -76,9 +75,8 @@ public static Job pollJob(final Bigquery.Jobs.Get request, final long interval) throws IOException, InterruptedException { Job job = request.execute(); while (!job.getStatus().getState().equals("DONE")) { - System.out.println("Job is " - + job.getStatus().getState() - + " waiting " + interval + " milliseconds..."); + System.out.println( + "Job is " + job.getStatus().getState() + " waiting " + interval + " milliseconds..."); Thread.sleep(interval); job = request.execute(); } @@ -165,9 +163,10 @@ public void remove() { public static TableSchema loadSchema(final Reader schemaSource) { TableSchema sourceSchema = new TableSchema(); - List fields = (new Gson()) - .>fromJson(schemaSource, - (new ArrayList()).getClass()); + List fields = + (new Gson()) + .>fromJson( + schemaSource, (new ArrayList()).getClass()); sourceSchema.setFields(fields); diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java index 736e5c31bbb..03316dddc05 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java @@ -26,7 +26,7 @@ /** * Sample of how to Export Cloud Data. */ -public class ExportDataCloudStorageSample { +public class ExportDataCloudStorageSample { /** * Protected constructor since this is a collection of static functions. */ @@ -42,8 +42,7 @@ protected ExportDataCloudStorageSample() { * @throws InterruptedException Should never be thrown. */ // [START main] - public static void main(final String[] args) - throws IOException, InterruptedException { + public static void main(final String[] args) throws IOException, InterruptedException { Scanner scanner = new Scanner(System.in); System.out.println("Enter your project id: "); String projectId = scanner.nextLine(); @@ -51,11 +50,9 @@ public static void main(final String[] args) String datasetId = scanner.nextLine(); System.out.println("Enter your table id: "); String tableId = scanner.nextLine(); - System.out.println("Enter the Google Cloud Storage Path to which you'd " - + "like to export: "); + System.out.println("Enter the Google Cloud Storage Path to which you'd " + "like to export: "); String cloudStoragePath = scanner.nextLine(); - System.out.println("Enter how often to check if your job is complete " - + "(milliseconds): "); + System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); long interval = scanner.nextLong(); scanner.close(); @@ -79,30 +76,33 @@ public static void run( final String projectId, final String datasetId, final String tableId, - final long interval) throws IOException, InterruptedException { + final long interval) + throws IOException, InterruptedException { - Bigquery bigquery = BigqueryServiceFactory.getService(); + Bigquery bigquery = BigQueryServiceFactory.getService(); - Job extractJob = extractJob( - bigquery, - cloudStoragePath, - new TableReference() - .setProjectId(projectId) - .setDatasetId(datasetId) - .setTableId(tableId)); + Job extractJob = + extractJob( + bigquery, + cloudStoragePath, + new TableReference() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId)); - Bigquery.Jobs.Get getJob = bigquery.jobs().get( - extractJob.getJobReference().getProjectId(), - extractJob.getJobReference().getJobId()); + Bigquery.Jobs.Get getJob = + bigquery + .jobs() + .get( + extractJob.getJobReference().getProjectId(), + extractJob.getJobReference().getJobId()); - BigqueryUtils.pollJob(getJob, interval); + BigQueryUtils.pollJob(getJob, interval); System.out.println("Export is Done!"); - } // [END run] - /** * A job that extracts data from a table. * @param bigquery Bigquery service to use @@ -113,16 +113,17 @@ public static void run( */ // [START extract_job] public static Job extractJob( - final Bigquery bigquery, - final String cloudStoragePath, - final TableReference table) throws IOException { + final Bigquery bigquery, final String cloudStoragePath, final TableReference table) + throws IOException { - JobConfigurationExtract extract = new JobConfigurationExtract() - .setSourceTable(table) - .setDestinationUri(cloudStoragePath); + JobConfigurationExtract extract = + new JobConfigurationExtract().setSourceTable(table).setDestinationUri(cloudStoragePath); - return bigquery.jobs().insert(table.getProjectId(), - new Job().setConfiguration(new JobConfiguration().setExtract(extract))) + return bigquery + .jobs() + .insert( + table.getProjectId(), + new Job().setConfiguration(new JobConfiguration().setExtract(extract))) .execute(); } // [END extract_job] diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java index 8cdb96a6489..b4549b62240 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Scanner; - /** * Example of authorizing with Bigquery and reading from a public dataset. * @@ -61,7 +60,8 @@ public static Bigquery createAuthorizedClient() throws IOException { } return new Bigquery.Builder(transport, jsonFactory, credential) - .setApplicationName("Bigquery Samples").build(); + .setApplicationName("Bigquery Samples") + .build(); } // [END build_service] @@ -77,15 +77,16 @@ public static Bigquery createAuthorizedClient() throws IOException { */ private static List executeQuery(String querySql, Bigquery bigquery, String projectId) throws IOException { - QueryResponse query = bigquery.jobs().query( - projectId, - new QueryRequest().setQuery(querySql)) - .execute(); + QueryResponse query = + bigquery.jobs().query(projectId, new QueryRequest().setQuery(querySql)).execute(); // Execute it - GetQueryResultsResponse queryResult = bigquery.jobs().getQueryResults( - query.getJobReference().getProjectId(), - query.getJobReference().getJobId()).execute(); + GetQueryResultsResponse queryResult = + bigquery + .jobs() + .getQueryResults( + query.getJobReference().getProjectId(), query.getJobReference().getJobId()) + .execute(); return queryResult.getRows(); } @@ -133,11 +134,14 @@ public static void main(String[] args) throws IOException { // Create a new Bigquery client authorized via Application Default Credentials. Bigquery bigquery = createAuthorizedClient(); - List rows = executeQuery("SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words " - + "FROM [publicdata:samples.shakespeare]", bigquery, projectId); + List rows = + executeQuery( + "SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words " + + "FROM [publicdata:samples.shakespeare]", + bigquery, + projectId); printResults(rows); } - } // [END all] diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java index 854e1b6e6ca..0334a2f2eda 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java @@ -50,9 +50,10 @@ public static void main(String[] args) throws IOException, InterruptedException } String projectId = args[0]; - Bigquery bigquery = BigqueryServiceFactory.getService(); - String query = "SELECT TOP( title, 10) as title, COUNT(*) as revision_count " - + "FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;"; + Bigquery bigquery = BigQueryServiceFactory.getService(); + String query = + "SELECT TOP( title, 10) as title, COUNT(*) as revision_count " + + "FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;"; System.out.println(); System.out.println("----- Running the asynchronous query and printing it to stdout."); @@ -120,8 +121,9 @@ public static void listProjects(Bigquery bigquery) throws IOException { * @param query A String containing a BigQuery SQL statement * @param out A PrintStream for output, normally System.out */ - static void runQueryRpcAndPrint(Bigquery bigquery, String projectId, String query, - PrintStream out) throws IOException, InterruptedException { + static void runQueryRpcAndPrint( + Bigquery bigquery, String projectId, String query, PrintStream out) + throws IOException, InterruptedException { QueryRequest queryRequest = new QueryRequest().setQuery(query); QueryResponse queryResponse = bigquery.jobs().query(projectId, queryRequest).execute(); if (queryResponse.getJobComplete()) { @@ -133,9 +135,12 @@ static void runQueryRpcAndPrint(Bigquery bigquery, String projectId, String quer // This loop polls until results are present, then loops over result pages. String pageToken = null; while (true) { - GetQueryResultsResponse queryResults = bigquery.jobs() - .getQueryResults(projectId, queryResponse.getJobReference().getJobId()) - .setPageToken(pageToken).execute(); + GetQueryResultsResponse queryResults = + bigquery + .jobs() + .getQueryResults(projectId, queryResponse.getJobReference().getJobId()) + .setPageToken(pageToken) + .execute(); if (queryResults.getJobComplete()) { printRows(queryResults.getRows(), out); pageToken = queryResults.getPageToken(); diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java index 28367529a68..b3ecb26c149 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java @@ -37,8 +37,7 @@ public class LoadDataCsvSample { /** * Protected constructor since this is a collection of static methods. */ - protected LoadDataCsvSample() { - } + protected LoadDataCsvSample() {} /** * Cli tool to load data from a CSV into Bigquery. @@ -47,8 +46,7 @@ protected LoadDataCsvSample() { * @throws InterruptedException InterruptedException */ // [START main] - public static void main(final String[] args) - throws IOException, InterruptedException { + public static void main(final String[] args) throws IOException, InterruptedException { Scanner scanner = new Scanner(System.in); System.out.println("Enter your project id: "); String projectId = scanner.nextLine(); @@ -56,19 +54,17 @@ public static void main(final String[] args) String datasetId = scanner.nextLine(); System.out.println("Enter your table id: "); String tableId = scanner.nextLine(); - System.out.println("Enter the Google Cloud Storage Path to the data " - + "you'd like to load: "); + System.out.println("Enter the Google Cloud Storage Path to the data " + "you'd like to load: "); String cloudStoragePath = scanner.nextLine(); System.out.println("Enter the filepath to your schema: "); String sourceSchemaPath = scanner.nextLine(); - - System.out.println("Enter how often to check if your job is complete " - + "(milliseconds): "); + System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); long interval = scanner.nextLong(); scanner.close(); - run(cloudStoragePath, + run( + cloudStoragePath, projectId, datasetId, tableId, @@ -95,28 +91,29 @@ public static void run( final String datasetId, final String tableId, final Reader schemaSource, - final long interval) throws IOException, InterruptedException { - - Bigquery bigquery = BigqueryServiceFactory.getService(); + final long interval) + throws IOException, InterruptedException { + Bigquery bigquery = BigQueryServiceFactory.getService(); - Job loadJob = loadJob( - bigquery, - cloudStoragePath, - new TableReference() - .setProjectId(projectId) - .setDatasetId(datasetId) - .setTableId(tableId), - BigqueryUtils.loadSchema(schemaSource)); + Job loadJob = + loadJob( + bigquery, + cloudStoragePath, + new TableReference() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId), + BigQueryUtils.loadSchema(schemaSource)); - Bigquery.Jobs.Get getJob = bigquery.jobs().get( - loadJob.getJobReference().getProjectId(), - loadJob.getJobReference().getJobId()); + Bigquery.Jobs.Get getJob = + bigquery + .jobs() + .get(loadJob.getJobReference().getProjectId(), loadJob.getJobReference().getJobId()); - BigqueryUtils.pollJob(getJob, interval); + BigQueryUtils.pollJob(getJob, interval); System.out.println("Load is Done!"); - } // [END run] @@ -134,15 +131,19 @@ public static Job loadJob( final Bigquery bigquery, final String cloudStoragePath, final TableReference table, - final TableSchema schema) throws IOException { - - JobConfigurationLoad load = new JobConfigurationLoad() - .setDestinationTable(table) - .setSchema(schema) - .setSourceUris(Collections.singletonList(cloudStoragePath)); - - return bigquery.jobs().insert(table.getProjectId(), - new Job().setConfiguration(new JobConfiguration().setLoad(load))) + final TableSchema schema) + throws IOException { + + JobConfigurationLoad load = + new JobConfigurationLoad() + .setDestinationTable(table) + .setSchema(schema) + .setSourceUris(Collections.singletonList(cloudStoragePath)); + + return bigquery + .jobs() + .insert( + table.getProjectId(), new Job().setConfiguration(new JobConfiguration().setLoad(load))) .execute(); } // [END load_job] diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java index 3f2aed4b747..ef37f042aab 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Scanner; - /** * Example of Bigquery Streaming. */ @@ -38,9 +37,7 @@ public class StreamingSample { /** * Empty constructor since this is just a collection of static methods. */ - protected StreamingSample() { - } - + protected StreamingSample() {} /** * Command line that demonstrates Bigquery streaming. @@ -59,15 +56,12 @@ public static void main(final String[] args) throws IOException { String tableId = scanner.nextLine(); scanner.close(); - System.out.println("Enter JSON to stream to BigQuery: \n" - + "Press End-of-stream (CTRL-D) to stop"); + System.out.println( + "Enter JSON to stream to BigQuery: \n" + "Press End-of-stream (CTRL-D) to stop"); JsonReader fromCli = new JsonReader(new InputStreamReader(System.in)); - Iterator responses = run(projectId, - datasetId, - tableId, - fromCli); + Iterator responses = run(projectId, datasetId, tableId, fromCli); while (responses.hasNext()) { System.out.println(responses.next()); @@ -77,7 +71,6 @@ public static void main(final String[] args) throws IOException { } // [END main] - /** * Run the bigquery ClI. * @@ -90,13 +83,11 @@ public static void main(final String[] args) throws IOException { * @throws InterruptedException Should never be thrown */ // [START run] - public static Iterator run(final String projectId, - final String datasetId, - final String tableId, - final JsonReader rows) throws IOException { - + public static Iterator run( + final String projectId, final String datasetId, final String tableId, final JsonReader rows) + throws IOException { - final Bigquery bigquery = BigqueryServiceFactory.getService(); + final Bigquery bigquery = BigQueryServiceFactory.getService(); final Gson gson = new Gson(); rows.beginArray(); @@ -123,10 +114,10 @@ public boolean hasNext() { */ public TableDataInsertAllResponse next() { try { - Map rowData = gson.>fromJson( - rows, - (new HashMap()).getClass()); - return streamRow(bigquery, + Map rowData = + gson.>fromJson(rows, (new HashMap()).getClass()); + return streamRow( + bigquery, projectId, datasetId, tableId, @@ -142,9 +133,7 @@ public TableDataInsertAllResponse next() { public void remove() { this.next(); } - }; - } // [END run] @@ -160,18 +149,22 @@ public void remove() { * @throws IOException ioexception */ // [START streamRow] - public static TableDataInsertAllResponse streamRow(final Bigquery bigquery, + public static TableDataInsertAllResponse streamRow( + final Bigquery bigquery, final String projectId, final String datasetId, final String tableId, - final TableDataInsertAllRequest.Rows row) throws IOException { - - return bigquery.tabledata().insertAll( - projectId, - datasetId, - tableId, - new TableDataInsertAllRequest().setRows( - Collections.singletonList(row))).execute(); + final TableDataInsertAllRequest.Rows row) + throws IOException { + + return bigquery + .tabledata() + .insertAll( + projectId, + datasetId, + tableId, + new TableDataInsertAllRequest().setRows(Collections.singletonList(row))) + .execute(); } // [END streamRow] } diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java index 497f719c4a9..3bfb30d9f64 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java @@ -28,13 +28,10 @@ */ public class SyncQuerySample { - /** * Protected because this is a collection of static methods. */ - protected SyncQuerySample() { - - } + protected SyncQuerySample() {} //[START main] /** @@ -43,27 +40,25 @@ protected SyncQuerySample() { * @param args args * @throws IOException ioexceptino */ - public static void main(final String[] args) - throws IOException { + public static void main(final String[] args) throws IOException { Scanner scanner = new Scanner(System.in); System.out.println("Enter your project id: "); String projectId = scanner.nextLine(); System.out.println("Enter your query string: "); String queryString = scanner.nextLine(); - System.out.println("Enter how long to wait for the query to complete" - + " (in milliseconds):\n " - + "(if longer than 10 seconds, use an asynchronous query)"); + System.out.println( + "Enter how long to wait for the query to complete" + + " (in milliseconds):\n " + + "(if longer than 10 seconds, use an asynchronous query)"); long waitTime = scanner.nextLong(); scanner.close(); - Iterator pages = run(projectId, queryString, - waitTime); + Iterator pages = run(projectId, queryString, waitTime); while (pages.hasNext()) { - BigqueryUtils.printRows(pages.next().getRows(), System.out); + BigQueryUtils.printRows(pages.next().getRows(), System.out); } } // [END main] - /** * Perform the given query using the synchronous api. * @@ -74,27 +69,27 @@ public static void main(final String[] args) * @throws IOException ioexception */ // [START run] - public static Iterator run(final String projectId, - final String queryString, - final long waitTime) throws IOException { - Bigquery bigquery = BigqueryServiceFactory.getService(); + public static Iterator run( + final String projectId, final String queryString, final long waitTime) throws IOException { + Bigquery bigquery = BigQueryServiceFactory.getService(); //Wait until query is done with 10 second timeout, at most 5 retries on error - QueryResponse query = bigquery.jobs().query( - projectId, - new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString)) - .execute(); + QueryResponse query = + bigquery + .jobs() + .query(projectId, new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString)) + .execute(); //Make a request to get the results of the query //(timeout is zero since job should be complete) - GetQueryResults getRequest = bigquery.jobs().getQueryResults( - query.getJobReference().getProjectId(), - query.getJobReference().getJobId()); - + GetQueryResults getRequest = + bigquery + .jobs() + .getQueryResults( + query.getJobReference().getProjectId(), query.getJobReference().getJobId()); - return BigqueryUtils.getPages(getRequest); + return BigQueryUtils.getPages(getRequest); } // [END run] - } diff --git a/bigquery/src/main/resources/constants.json b/bigquery/src/main/resources/constants.json deleted file mode 100644 index a1faa665d35..00000000000 --- a/bigquery/src/main/resources/constants.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "projectId": "cloud-samples-tests", - "datasetId": "test_dataset_java", - "currentTableId": "test_table_java", - "newTableId": "test_table_java_2", - "cloudStorageInputUri": "gs://cloud-samples-tests/data.csv", - "cloudStorageOutputUri": "gs://cloud-samples-tests/output.csv", - "query": "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;" -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java index c6c886034b4..a19c98b9f5c 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java @@ -20,29 +20,25 @@ import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.cloud.bigquery.samples.AsyncQuerySample; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.Iterator; /** * Tests for asynchronous query sample. */ -public class AsyncQuerySampleTest extends BigquerySampleTest{ - - public AsyncQuerySampleTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException { - super(); - } +@RunWith(JUnit4.class) +public class AsyncQuerySampleTest { @Test public void testInteractive() throws IOException, InterruptedException { Iterator pages = - AsyncQuerySample.run(CONSTANTS.getProjectId(), CONSTANTS.getQuery(), false, 5000); + AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, false, 5000); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); } @@ -52,7 +48,7 @@ public void testInteractive() throws IOException, InterruptedException { @Ignore // Batches can take up to 3 hours to run, probably shouldn't use this public void testBatch() throws IOException, InterruptedException { Iterator pages = - AsyncQuerySample.run(CONSTANTS.getProjectId(), CONSTANTS.getQuery(), true, 5000); + AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, true, 5000); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); } diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java deleted file mode 100644 index 047f9b249d5..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import com.google.cloud.bigquery.samples.BigqueryUtils; -import com.google.gson.Gson; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; - -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.io.InputStreamReader; - -/** - * Superclass for tests for samples. - */ -public class BigquerySampleTest extends BigqueryUtils { - - protected static class Constants { - private String projectId; - private String datasetId; - private String currentTableId; - private String newTableId; - private String cloudStorageInputUri; - private String cloudStorageOutputUri; - private String query; - - public String getProjectId() { - return projectId; - } - - public String getDatasetId() { - return datasetId; - } - - public String getCurrentTableId() { - return currentTableId; - } - - public String getNewTableId() { - return newTableId; - } - - public String getQuery() { - return query; - } - - public String getCloudStorageOutputUri() { - return cloudStorageOutputUri; - } - - public String getCloudStorageInputUri() { - return cloudStorageInputUri; - } - } - - @SuppressWarnings("checkstyle:abbreviationaswordinname") - protected static Constants CONSTANTS = null; - - protected BigquerySampleTest() - throws JsonSyntaxException, JsonIOException, FileNotFoundException { - if (CONSTANTS == null) { - InputStream is = this.getClass().getResourceAsStream("/constants.json"); - CONSTANTS = (new Gson()).fromJson(new InputStreamReader(is), Constants.class); - } - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java new file mode 100644 index 00000000000..26f9a64d16d --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http=//www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.bigquery.samples.test; + +public class Constants { + public static final String PROJECT_ID = "cloud-samples-tests"; + public static final String DATASET_ID = "test_dataset_java"; + public static final String CURRENT_TABLE_ID = "test_table_java"; + public static final String NEW_TABLE_ID = "test_table_java_2"; + public static final String CLOUD_STORAGE_INPUT_URI = "gs://cloud-samples-tests/data.csv"; + public static final String CLOUD_STORAGE_OUTPUT_URI = "gs://cloud-samples-tests/output.csv"; + public static final String QUERY = + "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;"; +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java index 3de3fdd4c58..c05e6b17eb0 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java @@ -17,31 +17,26 @@ package com.google.cloud.bigquery.samples.test; import com.google.cloud.bigquery.samples.ExportDataCloudStorageSample; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -import java.io.FileNotFoundException; import java.io.IOException; /** * Tests for export data Cloud Storage sample. */ -public class ExportDataCloudStorageSampleTest extends BigquerySampleTest { - - public ExportDataCloudStorageSampleTest() - throws JsonSyntaxException, JsonIOException, FileNotFoundException { - super(); - } +@RunWith(JUnit4.class) +public class ExportDataCloudStorageSampleTest { @Test public void testExportData() throws IOException, InterruptedException { ExportDataCloudStorageSample.run( - CONSTANTS.getCloudStorageOutputUri(), - CONSTANTS.getProjectId(), - CONSTANTS.getDatasetId(), - CONSTANTS.getCurrentTableId(), + Constants.CLOUD_STORAGE_OUTPUT_URI, + Constants.PROJECT_ID, + Constants.DATASET_ID, + Constants.CURRENT_TABLE_ID, 5000L); } } diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java index 0f688239cee..2a4babb4c1d 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java @@ -23,27 +23,24 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; - /** * Test for GettingStarted.java */ -public class GettingStartedTest extends BigquerySampleTest { +@RunWith(JUnit4.class) +public class GettingStartedTest { private static final PrintStream REAL_OUT = System.out; private static final PrintStream REAL_ERR = System.err; private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); - public GettingStartedTest() throws FileNotFoundException { - super(); - } - @Before public void setUp() { System.setOut(new PrintStream(stdout)); @@ -58,7 +55,7 @@ public void tearDown() { @Test public void testSyncQuery() throws IOException { - GettingStarted.main(new String[] { CONSTANTS.getProjectId() }); + GettingStarted.main(new String[] {Constants.PROJECT_ID}); String out = stdout.toString(); assertThat(out).named("stdout").containsMatch("Query Results:"); assertThat(out).named("stdout").contains("hamlet"); diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java index d4ebaeac2b9..6668f9b277a 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java @@ -23,26 +23,24 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; import java.io.PrintStream; import java.lang.Exception; /** * Unit tests for {@link ListDatasetsProjects}. */ -public class ListDatasetsProjectsTest extends BigquerySampleTest { +@RunWith(JUnit4.class) +public class ListDatasetsProjectsTest { private static final PrintStream REAL_OUT = System.out; private static final PrintStream REAL_ERR = System.err; private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); - public ListDatasetsProjectsTest() throws FileNotFoundException { - super(); - } - @Before public void setUp() { System.setOut(new PrintStream(stdout)); @@ -63,7 +61,7 @@ public void testUsage() throws Exception { @Test public void testMain() throws Exception { - ListDatasetsProjects.main(new String[] { CONSTANTS.getProjectId() }); + ListDatasetsProjects.main(new String[] {Constants.PROJECT_ID}); String out = stdout.toString(); assertThat(out).named("stdout").contains("Running the asynchronous query"); assertThat(out).named("stdout").containsMatch("George W. Bush, [0-9]+"); diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java index c87626cadf0..c33201e71a4 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java @@ -17,37 +17,30 @@ package com.google.cloud.bigquery.samples.test; import com.google.cloud.bigquery.samples.LoadDataCsvSample; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; /** * Tests for sample that loads data from CSV. */ -public class LoadDataCsvSampleTest extends BigquerySampleTest { - - public LoadDataCsvSampleTest() - throws JsonSyntaxException, JsonIOException, FileNotFoundException { - super(); - } +@RunWith(JUnit4.class) +public class LoadDataCsvSampleTest { @Test public void testLoadData() throws IOException, InterruptedException { InputStreamReader is = new InputStreamReader(LoadDataCsvSample.class.getResourceAsStream("/schema.json")); LoadDataCsvSample.run( - CONSTANTS.getCloudStorageInputUri(), - CONSTANTS.getProjectId(), - CONSTANTS.getDatasetId(), - CONSTANTS.getNewTableId(), + Constants.CLOUD_STORAGE_INPUT_URI, + Constants.PROJECT_ID, + Constants.DATASET_ID, + Constants.NEW_TABLE_ID, is, 5000L); } - - } diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java index b0624f23c01..01a6bd8f42a 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java @@ -20,13 +20,12 @@ import com.google.api.services.bigquery.model.TableDataInsertAllResponse; import com.google.cloud.bigquery.samples.StreamingSample; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.util.Iterator; @@ -34,24 +33,18 @@ /** * Tests for streaming sample. */ -public class StreamingSampleTest extends BigquerySampleTest { - - public StreamingSampleTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException { - super(); - } +@RunWith(JUnit4.class) +public class StreamingSampleTest { @Test public void testStream() throws IOException { JsonReader json = new JsonReader( new InputStreamReader( - BigquerySampleTest.class.getResourceAsStream("/streamrows.json"))); + StreamingSampleTest.class.getResourceAsStream("/streamrows.json"))); Iterator response = StreamingSample.run( - CONSTANTS.getProjectId(), - CONSTANTS.getDatasetId(), - CONSTANTS.getCurrentTableId(), - json); + Constants.PROJECT_ID, Constants.DATASET_ID, Constants.CURRENT_TABLE_ID, json); while (response.hasNext()) { assertThat(response.next()).isNotEmpty(); diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java index de52abb1f3d..f8a9441bc05 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java @@ -20,32 +20,24 @@ import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.cloud.bigquery.samples.SyncQuerySample; -import com.google.gson.JsonIOException; -import com.google.gson.JsonSyntaxException; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.Iterator; /** * Tests for synchronous query sample. */ -public class SyncQuerySampleTest extends BigquerySampleTest { - - public SyncQuerySampleTest() - throws JsonSyntaxException, JsonIOException, FileNotFoundException { - super(); - } +@RunWith(JUnit4.class) +public class SyncQuerySampleTest { @Test public void testSyncQuery() throws IOException { Iterator pages = - SyncQuerySample.run( - CONSTANTS.getProjectId(), - CONSTANTS.getQuery(), - 10000); + SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); }