Skip to content

Commit

Permalink
feat: add integration test to illustrate minimum set of hbase depende…
Browse files Browse the repository at this point in the history 
…ncies required for bigtable-hbase-2x (#4339)
  • Loading branch information
@igorbernstein2
igorbernstein2 authored Apr 10, 2024
1 parent 968884b commit c1a1a22
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 53 deletions.
4 changes: 2 additions & 2 deletions bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase1.version}</version>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
Expand Down Expand Up @@ -105,7 +105,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase1.version}</version>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
Expand Down
255 changes: 236 additions & 19 deletions bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,241 @@ limitations under the License.
<google.bigtable.connection.impl>com.google.cloud.bigtable.hbase2_x.BigtableConnection</google.bigtable.connection.impl>
<google.bigtable.async.connection.impl>org.apache.hadoop.hbase.client.BigtableAsyncConnection</google.bigtable.async.connection.impl>
<test.timeout>1800</test.timeout>

<!-- System property `int-test-2x-deps.mode` selects which dependency configuration is active.
It can be be one of 3 states: unset, `cbt-shaded-hbase-unshaded` and `cbt-shaded-hbase-minimal-unshaded`
The default (unset) will use unshaded bigtable-hbase adapter and hbase-shaded-client.
The other 2 modes will use bigtable-hbase-2x-shaded artifact and either the full hbase-client
or a partial hbase-client, that only includes the bare minimum of hbase-client.
-->

<!-- This controls the hadoop version in the `cbt-shaded-hbase-minimal-unshaded` mode -->
<minimal-dep-hadoop.version>3.3.6</minimal-dep-hadoop.version>
</properties>

<profiles>
<!-- NOTE: this profile is controlled via `int-test-2x-deps.mode` property -->
<profile>
<id>cbt-unshaded-hbase-shaded-deps</id>
<activation>
<property>
<!-- On by default -->
<name>!int-test-2x-deps.mode</name>
</property>
</activation>

<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase.version}</version>
</dependency>

<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-2.x</artifactId>
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>

<!-- NOTE: this profile is controlled via `int-test-2x-deps.mode` property -->
<profile>
<id>cbt-shaded-hbase-unshaded-deps</id>

<activation>
<property>
<name>int-test-2x-deps.mode</name>
<value>cbt-shaded-hbase-unshaded</value>
</property>
</activation>

<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>

<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-2.x-shaded</artifactId>
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>

<!-- NOTE: this profile is controlled via `int-test-2x-deps.mode` property -->
<profile>
<id>cbt-shaded-hbase-minimal-unshaded-deps</id>

<activation>
<property>
<name>int-test-2x-deps.mode</name>
<value>cbt-shaded-hbase-minimal-unshaded</value>
</property>
</activation>

<!-- HBase doesnt have an interface/value only artifact, and the client jar pulls in a lot
of extra deps that bigtable doesn't need.
This profile starts with hbase-client w/o transitive deps and adds all the necessary deps
to allow the bigtable-hbase to properly function. This was done experimentally by adding
missing deps until the integration tests pass.
The HBase & Hadoop have large set of transitive deps, so we explicitly exclude all of them
add them manually. All hbase deps are pinned to hbase < 2.0. Everything else is using the latest available version.
-->

<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-unsafe</artifactId>
<version>4.1.4</version>
</dependency>

<!-- HBase uses Configuration machinery from hadoop, the smallest dep we can get is hadoop-common -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${minimal-dep-hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${minimal-dep-hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-api</artifactId>
<version>1.34.1</version>
</dependency>

<!-- Pull in the minimal deps to get hadoop Configuration to work and update all deps to latest version -->
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
<version>6.5.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>

<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-miscellaneous</artifactId>
<version>4.1.4</version>
</dependency>
<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-protobuf</artifactId>
<version>4.1.4</version>
</dependency>

<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-2.x-shaded</artifactId>
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>

<!-- This is only necessary modules built in the java-bigtable-hbase repo. End user
applications don't need these exclusions.
Workaround MNG-5899 & MSHADE-206. Maven >= 3.3.0 doesn't use the dependency reduced
pom.xml files when invoking the build from a parent project. So we have to manually exclude
the dependencies. Note that this works in conjunction with the manually promoted dependencies
in bigtable-hbase-2.x-shaded/pom.xml -->
<exclusion>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-2.x</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
</exclusion>
<exclusion>
<groupId>io.opencensus</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>io.opentelemetry</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-census</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>


<profile>
<id>bigtableIntegrationTestH2</id>
<build>
Expand Down Expand Up @@ -199,20 +431,6 @@ limitations under the License.
</dependencyManagement>

<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase2.version}</version>
</dependency>

<!-- Project Modules -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>bigtable-hbase-2.x</artifactId>
<version>2.14.0-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<scope>test</scope>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>bigtable-hbase-integration-tests-common</artifactId>
Expand All @@ -221,8 +439,8 @@ limitations under the License.
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>org.apache.hbase</artifactId>
<groupId>*</groupId>
<groupId>org.apache.hbase</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand All @@ -242,15 +460,14 @@ limitations under the License.
<scope>test</scope>
</dependency>


<!-- Testing deps -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
<scope>test</scope>
</dependency>


<!-- Testing deps -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
Expand Down
32 changes: 0 additions & 32 deletions ...base-2.x-integration-tests/src/test/java/com/google/cloud/bigtable/hbase/TestFilters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,10 @@

import static com.google.cloud.bigtable.hbase.test_env.SharedTestEnvRule.COLUMN_FAMILY;

import com.google.cloud.bigtable.data.v2.models.Filters;
import com.google.cloud.bigtable.hbase.filter.BigtableFilter;
import com.google.cloud.bigtable.hbase.filter.TimestampRangeFilter;
import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
Expand Down Expand Up @@ -68,35 +65,6 @@ public void testTimestampRangeFilter() throws IOException {
table.close();
}

// This test can't run against the minicluster because its a Bigtable extension
@Category(KnownHBaseGap.class)
@Test
public void testBigtableFilter() throws IOException {
if (!sharedTestEnv.isBigtable()) {
return;
}

byte[] rowKey = dataHelper.randomData("cbt-filter-");
byte[] qualA = Bytes.toBytes("a");
byte[] qualB = Bytes.toBytes("b");
byte[] valA = dataHelper.randomData("a");
byte[] valB = dataHelper.randomData("b");

try (Table table = getDefaultTable()) {
table.put(
new Put(rowKey)
.addColumn(COLUMN_FAMILY, qualA, valA)
.addColumn(COLUMN_FAMILY, qualB, valB));

Filters.Filter qualAFilter = Filters.FILTERS.qualifier().exactMatch(new String(qualA));
BigtableFilter bigtableFilter = new BigtableFilter(qualAFilter);
Result result = table.get(new Get(rowKey).setFilter(bigtableFilter));

Assert.assertEquals(1, result.size());
Assert.assertTrue(CellUtil.matchingValue(result.rawCells()[0], valA));
}
}

/**
* This test case is used to validate TimestampRangeFilter with Integer.MAX_VALUE #1552
*
Expand Down

0 comments on commit c1a1a22

Please sign in to comment.