From 2c6a7aa082d49db1fa835730df206287c6d5b187 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 27 Jan 2017 14:12:51 -0800 Subject: [PATCH 1/2] BigQuery REST samples to label datasets and tables. These samples use the HTTP client library. Neither the Cloud nor API client libraries support the table labeling feature yet. The Cloud client library will not support it for a while, since this is a beta feature. I made an agreement with the Cloud client libraries PM that I can mix client libraries and raw HTTP samples. --- bigquery/pom.xml | 23 +- bigquery/rest/README.md | 40 ++++ bigquery/rest/pom.xml | 100 +++++++++ .../com/example/bigquery/LabelsSample.java | 210 ++++++++++++++++++ .../java/com/example/bigquery/Constants.java | 23 ++ .../com/example/bigquery/LabelsSampleIT.java | 69 ++++++ pom.xml | 1 + 7 files changed, 449 insertions(+), 17 deletions(-) create mode 100644 bigquery/rest/README.md create mode 100644 bigquery/rest/pom.xml create mode 100644 bigquery/rest/src/main/java/com/example/bigquery/LabelsSample.java create mode 100644 bigquery/rest/src/test/java/com/example/bigquery/Constants.java create mode 100644 bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java diff --git a/bigquery/pom.xml b/bigquery/pom.xml index 352676b0206..739554f283d 100644 --- a/bigquery/pom.xml +++ b/bigquery/pom.xml @@ -75,23 +75,12 @@ - src/main/java - - - src/main/resources - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - 5 - 5 - - - + src/main/java + + + src/main/resources + + diff --git a/bigquery/rest/README.md b/bigquery/rest/README.md new file mode 100644 index 00000000000..486e6642c1e --- /dev/null +++ b/bigquery/rest/README.md @@ -0,0 +1,40 @@ +# Getting Started with BigQuery with the REST API + +Google's BigQuery Service features a REST-based API that allows developers to +create applications to run ad-hoc queries on massive datasets. These sample +Java applications demonstrate how to access the BigQuery API directly using the +Google HTTP Client. + +## Quickstart + +Install [Maven](http://maven.apache.org/). + +Build your project with: + + mvn clean package -DskipTests + +You can then run a given `ClassName` via: + + mvn exec:java -Dexec.mainClass=com.example.bigquery.ClassName \ + -Dexec.args="any arguments to the app" + +### Labeling a dataset + +[Label a dataset](https://cloud.google.com/bigquery/docs/labeling-datasets). + + mvn exec:java -Dexec.mainClass=com.example.bigquery.LabelsSample \ + -Dexec.args="project-id dataset-id label-key label-value" + +## Products +- [Google BigQuery][2] + +## Language +- [Java][3] + +## Dependencies +- [Google HTTP Client Library for Java][4] + +[2]: https://cloud.google.com/bigquery +[3]: https://java.com +[4]: https://github.com/google/google-http-java-client + diff --git a/bigquery/rest/pom.xml b/bigquery/rest/pom.xml new file mode 100644 index 00000000000..462fef0f0b9 --- /dev/null +++ b/bigquery/rest/pom.xml @@ -0,0 +1,100 @@ + + + 4.0.0 + 1.0.0 + + com.google.cloud.samples + bigquery-rest-samples + jar + + + + doc-samples + com.google.cloud + 1.0.0 + ../.. + + + + UTF-8 + + + + + com.google.guava + guava + 20.0 + + + com.google.api-client + google-api-client + 1.22.0 + + + com.google.guava + guava-jdk5 + + + + + com.google.http-client + google-http-client + 1.22.0 + + + com.google.guava + guava-jdk5 + + + + + com.google.oauth-client + google-oauth-client + 1.22.0 + + + com.google.guava + guava-jdk5 + + + + + + + junit + junit + 4.12 + test + + + com.google.truth + truth + 0.31 + test + + + + + src/main/java + + + src/main/resources + + + + + diff --git a/bigquery/rest/src/main/java/com/example/bigquery/LabelsSample.java b/bigquery/rest/src/main/java/com/example/bigquery/LabelsSample.java new file mode 100644 index 00000000000..c84b7a0c143 --- /dev/null +++ b/bigquery/rest/src/main/java/com/example/bigquery/LabelsSample.java @@ -0,0 +1,210 @@ +/* + * Copyright 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpContent; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.http.json.JsonHttpContent; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.util.Key; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** Sample demonstrating labeling a BigQuery dataset or table. */ +public class LabelsSample { + + // [START label_dataset] + static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); + static final JsonFactory JSON_FACTORY = new JacksonFactory(); + + public static class Dataset { + @Key private Map labels; + + public Map getLabels() { + return this.labels; + } + + public Dataset addLabel(String key, String value) { + if (this.labels == null) { + this.labels = new HashMap<>(); + } + this.labels.put(key, value); + return this; + } + } + + /** + * Add or modify a label on a dataset. + * + *

See the BigQuery + * documentation. + */ + public static void labelDataset( + String projectId, String datasetId, String labelKey, String labelValue) throws IOException { + + // Authenticate requests using Google Application Default credentials. + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + credential = credential.createScoped(Arrays.asList("https://www.googleapis.com/auth/bigquery")); + + // Get a new access token. + // Note that access tokens have an expiration. You can reuse a token rather than requesting a + // new one if it is not yet expired. + credential.refreshToken(); + String accessToken = credential.getAccessToken(); + + // Set the content of the request. + Dataset dataset = new Dataset(); + dataset.addLabel(labelKey, labelValue); + HttpContent content = new JsonHttpContent(JSON_FACTORY, dataset); + + // Send the request to the BigQuery API. + String urlFormat = + "https://www.googleapis.com/bigquery/v2/projects/%s/datasets/%s" + + "?fields=labels&access_token=%s"; + GenericUrl url = new GenericUrl(String.format(urlFormat, projectId, datasetId, accessToken)); + HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(); + HttpRequest request = requestFactory.buildPostRequest(url, content); + request.setParser(JSON_FACTORY.createJsonObjectParser()); + + // Workaround for transports which do not support PATCH requests. + // See: http://stackoverflow.com/a/32503192/101923 + request.setHeaders(new HttpHeaders().set("X-HTTP-Method-Override", "PATCH")); + HttpResponse response = request.execute(); + + // Check for errors. + if (response.getStatusCode() != 200) { + throw new RuntimeException(response.getStatusMessage()); + } + + Dataset responseDataset = response.parseAs(Dataset.class); + System.out.printf( + "Updated label \"%s\" with value \"%s\"\n", + labelKey, responseDataset.getLabels().get(labelKey)); + } + // [END label_dataset] + + // [START label_table] + public static class Table { + @Key private Map labels; + + public Map getLabels() { + return this.labels; + } + + public Table addLabel(String key, String value) { + if (this.labels == null) { + this.labels = new HashMap<>(); + } + this.labels.put(key, value); + return this; + } + } + + /** + * Add or modify a label on a table. + * + *

See the BigQuery + * documentation. + */ + public static void labelTable( + String projectId, + String datasetId, + String tableId, + String labelKey, + String labelValue) + throws IOException { + + // Authenticate requests using Google Application Default credentials. + GoogleCredential credential = GoogleCredential.getApplicationDefault(); + credential = credential.createScoped(Arrays.asList("https://www.googleapis.com/auth/bigquery")); + + // Get a new access token. + // Note that access tokens have an expiration. You can reuse a token rather than requesting a + // new one if it is not yet expired. + credential.refreshToken(); + String accessToken = credential.getAccessToken(); + + // Set the content of the request. + Table table = new Table(); + table.addLabel(labelKey, labelValue); + HttpContent content = new JsonHttpContent(JSON_FACTORY, table); + + // Send the request to the BigQuery API. + String urlFormat = + "https://www.googleapis.com/bigquery/v2/projects/%s/datasets/%s/tables/%s" + + "?fields=labels&access_token=%s"; + GenericUrl url = + new GenericUrl(String.format(urlFormat, projectId, datasetId, tableId, accessToken)); + HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(); + HttpRequest request = requestFactory.buildPostRequest(url, content); + request.setParser(JSON_FACTORY.createJsonObjectParser()); + + // Workaround for transports which do not support PATCH requests. + // See: http://stackoverflow.com/a/32503192/101923 + request.setHeaders(new HttpHeaders().set("X-HTTP-Method-Override", "PATCH")); + HttpResponse response = request.execute(); + + // Check for errors. + if (response.getStatusCode() != 200) { + throw new RuntimeException(response.getStatusMessage()); + } + + Table responseTable = response.parseAs(Table.class); + System.out.printf( + "Updated label \"%s\" with value \"%s\"\n", + labelKey, responseTable.getLabels().get(labelKey)); + } + // [END label_table] + + public static void printUsage() { + System.err.println("Command expects 4 or 5 arguments:"); + System.err.println("\tproject dataset [table] key value"); + } + + public static void main(final String[] args) throws IOException, InterruptedException { + if (args.length != 4 && args.length != 5) { + printUsage(); + System.exit(1); + } + + if (args.length == 4) { + String projectId = args[0]; + String datasetId = args[1]; + String labelKey = args[2]; + String labelValue = args[3]; + labelDataset(projectId, datasetId, labelKey, labelValue); + } else { + String projectId = args[0]; + String datasetId = args[1]; + String tableId = args[2]; + String labelKey = args[3]; + String labelValue = args[4]; + labelTable(projectId, datasetId, tableId, labelKey, labelValue); + } + } +} diff --git a/bigquery/rest/src/test/java/com/example/bigquery/Constants.java b/bigquery/rest/src/test/java/com/example/bigquery/Constants.java new file mode 100644 index 00000000000..5422a2f5b74 --- /dev/null +++ b/bigquery/rest/src/test/java/com/example/bigquery/Constants.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http=//www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.example.bigquery; + +public class Constants { + public static final String PROJECT_ID = "cloud-samples-tests"; + public static final String DATASET_ID = "test_dataset_java"; + public static final String TABLE_ID = "test_table_java"; +} diff --git a/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java b/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java new file mode 100644 index 00000000000..26fea8e9f1b --- /dev/null +++ b/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** Integration tests for sample which labels datasets and tables. */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class LabelsSampleIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testLabelDataset() throws Exception { + LabelsSample.main( + new String[] {Constants.PROJECT_ID, Constants.DATASET_ID, "environment", "test"}); + String got = bout.toString(); + assertThat(got).contains("Updated label \"environment\" with value \"test\""); + } + + @Test + public void testLabelTable() throws Exception { + LabelsSample.main( + new String[] { + Constants.PROJECT_ID, + Constants.DATASET_ID, + Constants.TABLE_ID, + "data-owner", + "my-team"}); + String got = bout.toString(); + assertThat(got).contains("Updated label \"data-owner\" with value \"my-team\""); + } +} diff --git a/pom.xml b/pom.xml index 20c9254cb0f..26cc5708d72 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ bigquery bigquery/cloud-client + bigquery/rest compute/cmdline compute/error-reporting compute/mailjet From 26c61c672047fdd794636ae2138803e6411f28d1 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 30 Jan 2017 10:37:09 -0800 Subject: [PATCH 2/2] Use GOOGLE_CLOUD_PROJECT for BigQuery tests. This removes dependency on hard-coded project name for the BigQuery REST samples. --- .travis.yml | 1 + bigquery/rest/README.md | 22 ++++++++++++++----- .../java/com/example/bigquery/Constants.java | 5 ++--- .../com/example/bigquery/LabelsSampleIT.java | 6 +++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e48bda620a..6fb38ee5c51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ before_install: - openssl aes-256-cbc -K $encrypted_eb858daba67b_key -iv $encrypted_eb858daba67b_iv -in secrets.env.enc -out secrets.env -d && set +x && source secrets.env && set -x || true +- export GOOGLE_CLOUD_PROJECT=java-docs-samples-tests # Skip the install step, since Maven will download the dependencies we need # when the test build runs. # http://stackoverflow.com/q/31945809/101923 diff --git a/bigquery/rest/README.md b/bigquery/rest/README.md index 486e6642c1e..e8cc0058e5a 100644 --- a/bigquery/rest/README.md +++ b/bigquery/rest/README.md @@ -11,19 +11,31 @@ Install [Maven](http://maven.apache.org/). Build your project with: - mvn clean package -DskipTests + mvn clean package -DskipTests You can then run a given `ClassName` via: - mvn exec:java -Dexec.mainClass=com.example.bigquery.ClassName \ - -Dexec.args="any arguments to the app" + mvn exec:java -Dexec.mainClass=com.example.bigquery.ClassName \ + -Dexec.args="any arguments to the app" ### Labeling a dataset [Label a dataset](https://cloud.google.com/bigquery/docs/labeling-datasets). - mvn exec:java -Dexec.mainClass=com.example.bigquery.LabelsSample \ - -Dexec.args="project-id dataset-id label-key label-value" + mvn exec:java -Dexec.mainClass=com.example.bigquery.LabelsSample \ + -Dexec.args="project-id dataset-id label-key label-value" + +## Testing + +To run the tests for this sample, first set the `GOOGLE_CLOUD_PROJECT` +environment variable. The project should have a dataset named `test_dataset` +with a table named `test_table`. + + export GOOGLE_CLOUD_PROJECT=my-project + +Then run the tests with Maven. + + mvn clean verify ## Products - [Google BigQuery][2] diff --git a/bigquery/rest/src/test/java/com/example/bigquery/Constants.java b/bigquery/rest/src/test/java/com/example/bigquery/Constants.java index 5422a2f5b74..c2401339b62 100644 --- a/bigquery/rest/src/test/java/com/example/bigquery/Constants.java +++ b/bigquery/rest/src/test/java/com/example/bigquery/Constants.java @@ -17,7 +17,6 @@ package com.example.bigquery; public class Constants { - public static final String PROJECT_ID = "cloud-samples-tests"; - public static final String DATASET_ID = "test_dataset_java"; - public static final String TABLE_ID = "test_table_java"; + public static final String DATASET_ID = "test_dataset"; + public static final String TABLE_ID = "test_table"; } diff --git a/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java b/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java index 26fea8e9f1b..f9156da0320 100644 --- a/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java +++ b/bigquery/rest/src/test/java/com/example/bigquery/LabelsSampleIT.java @@ -33,9 +33,11 @@ public class LabelsSampleIT { private ByteArrayOutputStream bout; private PrintStream out; + private String projectId; @Before public void setUp() { + projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); bout = new ByteArrayOutputStream(); out = new PrintStream(bout); System.setOut(out); @@ -49,7 +51,7 @@ public void tearDown() { @Test public void testLabelDataset() throws Exception { LabelsSample.main( - new String[] {Constants.PROJECT_ID, Constants.DATASET_ID, "environment", "test"}); + new String[] {projectId, Constants.DATASET_ID, "environment", "test"}); String got = bout.toString(); assertThat(got).contains("Updated label \"environment\" with value \"test\""); } @@ -58,7 +60,7 @@ public void testLabelDataset() throws Exception { public void testLabelTable() throws Exception { LabelsSample.main( new String[] { - Constants.PROJECT_ID, + projectId, Constants.DATASET_ID, Constants.TABLE_ID, "data-owner",