docs(samples): add query external bigtable using permanent table (#764)

* docs(samples): add query external bigtable using permanent table

* docs(samples): address feedback

* docs(samples): address comments

samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example.bigquery;
+
+// [START bigquery_query_external_bigtable_perm]
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryException;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigtableColumn;
+import com.google.cloud.bigquery.BigtableColumnFamily;
+import com.google.cloud.bigquery.BigtableOptions;
+import com.google.cloud.bigquery.ExternalTableDefinition;
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.TableInfo;
+import com.google.cloud.bigquery.TableResult;
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.codec.binary.Base64;
+
+// Sample to queries an external bigtable data source using a permanent table
+public class QueryExternalBigtablePerm {
+
+  public static void main(String[] args) {
+    // TODO(developer): Replace these variables before running the sample.
+    String projectId = "MY_PROJECT_ID";
+    String bigtableInstanceId = "MY_INSTANCE_ID";
+    String bigtableTableName = "MY_BIGTABLE_NAME";
+    String bigqueryDatasetName = "MY_DATASET_NAME";
+    String bigqueryTableName = "MY_TABLE_NAME";
+    String sourceUri =
+        String.format(
+            "https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
+            projectId, bigtableInstanceId, bigtableTableName);
+    String query = String.format("SELECT * FROM %s ", bigqueryTableName);
+    queryExternalBigtablePerm(bigqueryDatasetName, bigqueryTableName, sourceUri, query);
+  }
+
+  public static void queryExternalBigtablePerm(
+      String datasetName, String tableName, String sourceUri, String query) {
+    try {
+      // Initialize client that will be used to send requests. This client only needs to be created
+      // once, and can be reused for multiple requests.
+      BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
+
+      BigtableColumnFamily.Builder statsSummary = BigtableColumnFamily.newBuilder();
+
+      // Configuring Columns
+      BigtableColumn connectedCell =
+          BigtableColumn.newBuilder()
+              .setQualifierEncoded(Base64.encodeBase64String("connected_cell".getBytes()))
+              .setFieldName("connected_cell")
+              .setType("STRING")
+              .setEncoding("TEXT")
+              .build();
+      BigtableColumn connectedWifi =
+          BigtableColumn.newBuilder()
+              .setQualifierEncoded(Base64.encodeBase64String("connected_wifi".getBytes()))
+              .setFieldName("connected_wifi")
+              .setType("STRING")
+              .setEncoding("TEXT")
+              .build();
+      BigtableColumn osBuild =
+          BigtableColumn.newBuilder()
+              .setQualifierEncoded(Base64.encodeBase64String("os_build".getBytes()))
+              .setFieldName("os_build")
+              .setType("STRING")
+              .setEncoding("TEXT")
+              .build();
+
+      // Configuring column family and columns
+      statsSummary
+          .setColumns(ImmutableList.of(connectedCell, connectedWifi, osBuild))
+          .setFamilyID("stats_summary")
+          .setOnlyReadLatest(true)
+          .setEncoding("TEXT")
+          .setType("STRING")
+          .build();
+
+      // Configuring BigtableOptions is optional.
+      BigtableOptions options =
+          BigtableOptions.newBuilder()
+              .setIgnoreUnspecifiedColumnFamilies(true)
+              .setReadRowkeyAsString(true)
+              .setColumnFamilies(ImmutableList.of(statsSummary.build()))
+              .build();
+
+      TableId tableId = TableId.of(datasetName, tableName);
+      // Create a permanent table linked to the Bigtable table
+      ExternalTableDefinition externalTable =
+          ExternalTableDefinition.newBuilder(sourceUri, options).build();
+      bigquery.create(TableInfo.of(tableId, externalTable));
+
+      // Example query
+      TableResult results = bigquery.query(QueryJobConfiguration.of(query));
+
+      results
+          .iterateAll()
+          .forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString())));
+
+      System.out.println("Query on external permanent table performed successfully.");
+    } catch (BigQueryException | InterruptedException e) {
+      System.out.println("Query not performed \n" + e.toString());
+    }
+  }
+}
+// [END bigquery_query_external_bigtable_perm]

samples/snippets/src/test/java/com/example/bigquery/QueryExternalBigtablePermIT.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example.bigquery;
+
+import static com.google.common.truth.Truth.assertThat;
+import static junit.framework.TestCase.assertNotNull;
+
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
+import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.models.BulkMutation;
+import com.google.cloud.bigtable.data.v2.models.Mutation;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class QueryExternalBigtablePermIT {
+
+  private final Logger log = Logger.getLogger(this.getClass().getName());
+  private static final String ID = UUID.randomUUID().toString().substring(0, 8);
+  private static final String TABLE_ID = "bigquery-samples-test" + ID;
+  private static final String COLUMN_FAMILY_NAME = "stats_summary";
+  private static final long TIMESTAMP = System.currentTimeMillis() * 1000;
+  private static final String CONNECTED_CELL = "connected_cell";
+  private static final String CONNECTED_WIFI = "connected_wifi";
+  private static final String OS_BUILD = "os_build";
+  private String tableName;
+  private ByteArrayOutputStream bout;
+  private PrintStream out;
+  private PrintStream originalPrintStream;
+
+  private static final String INSTANCE = requireEnvVar("BIGTABLE_TESTING_INSTANCE");
+  private static final String PROJECT = requireEnvVar("SAMPLES_TESTING_PROJECT");
+  private static final String BIGQUERY_DATASET_NAME = requireEnvVar("BIGQUERY_DATASET_NAME");
+
+  private static String requireEnvVar(String varName) {
+    String value = System.getenv(varName);
+    assertNotNull(
+        "Environment variable " + varName + " is required to perform these tests.",
+        System.getenv(varName));
+    return value;
+  }
+
+  @BeforeClass
+  public static void checkRequirements() {
+    requireEnvVar("GOOGLE_CLOUD_PROJECT");
+    requireEnvVar("BIGTABLE_TESTING_INSTANCE");
+    requireEnvVar("BIGQUERY_DATASET_NAME");
+  }
+
+  @Before
+  public void setUp() throws IOException {
+    // Create a test table
+    tableName = "EXTERNAL_TABLE_FROM_BIGTABLE_TEST_" + ID;
+    bout = new ByteArrayOutputStream();
+    out = new PrintStream(bout);
+    originalPrintStream = System.out;
+    System.setOut(out);
+
+    // create a temporary bigtable table.
+    try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
+      CreateTableRequest createTableRequest =
+          CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME);
+      client.createTable(createTableRequest);
+    }
+    // inserting temporary rows.
+    try (BigtableDataClient client = BigtableDataClient.create(PROJECT, INSTANCE)) {
+      BulkMutation bulkMutation =
+          BulkMutation.create(TABLE_ID)
+              .add(
+                  "phone#4c410523#20190501",
+                  Mutation.create()
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_CELL.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.003"))
+              .add(
+                  "phone#4c410523#20190502",
+                  Mutation.create()
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_CELL.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.004"))
+              .add(
+                  "phone#4c410523#20190505",
+                  Mutation.create()
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_CELL.getBytes()),
+                          TIMESTAMP,
+                          0)
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"))
+              .add(
+                  "phone#5c10102#20190501",
+                  Mutation.create()
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_CELL.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190401.002"))
+              .add(
+                  "phone#5c10102#20190502",
+                  Mutation.create()
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_CELL.getBytes()),
+                          TIMESTAMP,
+                          1)
+                      .setCell(
+                          COLUMN_FAMILY_NAME,
+                          ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
+                          TIMESTAMP,
+                          0)
+                      .setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"));
+
+      client.bulkMutateRows(bulkMutation);
+    }
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    // Clean up
+    try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
+      client.deleteTable(TABLE_ID);
+    }
+    DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName);
+    // restores print statements in the original method
+    System.out.flush();
+    System.setOut(originalPrintStream);
+    log.log(Level.INFO, bout.toString());
+  }
+
+  @Test
+  public void testQueryExternalBigtablePerm() {
+    String query = String.format("SELECT * FROM %s.%s ", BIGQUERY_DATASET_NAME, tableName);
+    String sourceUri =
+        String.format(
+            "https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
+            PROJECT, INSTANCE, TABLE_ID);
+    QueryExternalBigtablePerm.queryExternalBigtablePerm(
+        BIGQUERY_DATASET_NAME, tableName, sourceUri, query);
+    assertThat(bout.toString())
+        .contains("Query on external permanent table performed successfully.");
+  }
+}