From acd76083ebae5895d0d70bfc6e828c69ebffd8de Mon Sep 17 00:00:00 2001 From: Suryansh Agnihotri Date: Fri, 4 Sep 2020 17:04:38 +0530 Subject: [PATCH] Remove shading of thrift lib from submodule (#446) Shade thrift and all submodules into single jar in rubix-build module Co-authored-by: Suryansh Agnihotri --- .../release-0.3.17-(next-release).rst | 3 +- pom.xml | 1 + .../qubole/rubix/bookkeeper/BookKeeper.java | 2 +- .../rubix/bookkeeper/BookKeeperServer.java | 10 +- .../rubix/bookkeeper/HeartbeatService.java | 2 +- .../bookkeeper/LocalDataTransferServer.java | 2 +- .../validation/CachingValidator.java | 2 +- .../rubix/bookkeeper/BaseServerTest.java | 6 +- .../rubix/bookkeeper/TestBookKeeper.java | 2 +- .../bookkeeper/TestHeartbeatService.java | 6 +- .../validation/TestCachingValidator.java | 6 +- rubix-build/pom.xml | 116 ++++++++++++++++++ .../BookKeeperClientRFLibrary.java | 4 +- .../client/GetCacheMetricsRequest.java | 2 +- .../server/ContainerRequestServer.java | 2 +- .../rubix/health/TestBookKeeperHealth.java | 4 +- rubix-presto-shaded/pom.xml | 4 + rubix-spi/pom.xml | 4 - .../rubix/tests/TestGenerationNumber.java | 2 +- .../tests/TestNonLocalReadRequestChain.java | 2 +- 20 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 rubix-build/pom.xml diff --git a/docs/release/release_notes/release-0.3.17-(next-release).rst b/docs/release/release_notes/release-0.3.17-(next-release).rst index 198b9216..355885ce 100644 --- a/docs/release/release_notes/release-0.3.17-(next-release).rst +++ b/docs/release/release_notes/release-0.3.17-(next-release).rst @@ -4,4 +4,5 @@ Release 0.3.17 (next release) Fixes and Features ------------------ -* Added `total_system_source_mb_read` stat in detailed metrics to show total data read from source: during read + warmups \ No newline at end of file +* Added `total_system_source_mb_read` stat in detailed metrics to show total data read from source: during read + warmups +* Moved shading of Thrift classes into a common sub-module rubix-build. Project is now traversable in IDE and mvn test works at the root. Clients should now include rubix-build artifact instead of including sub-modules independently \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3d20433f..520d516e 100644 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,7 @@ assembly rubix-rpm rubix-presto-shaded + rubix-build diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java index 20c2abca..837cc24c 100644 --- a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java +++ b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java @@ -56,7 +56,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.DirectBufferPool; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import java.io.File; import java.io.FileNotFoundException; diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java index 8c68f097..f5ba8556 100644 --- a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java +++ b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java @@ -27,11 +27,11 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.apache.thrift.shaded.server.TServer; -import org.apache.thrift.shaded.server.TThreadPoolServer; -import org.apache.thrift.shaded.transport.TServerSocket; -import org.apache.thrift.shaded.transport.TServerTransport; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransportException; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java index 34452a8e..3ee6b8f5 100644 --- a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java +++ b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java @@ -28,7 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import java.io.IOException; import java.net.InetAddress; diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java index 417670be..84758972 100644 --- a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java +++ b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java @@ -36,7 +36,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import java.io.FileInputStream; import java.io.FileNotFoundException; diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java index 0b1d7d3b..b974f475 100644 --- a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java +++ b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java @@ -27,7 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import java.io.File; import java.io.IOException; diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java index 773ff09f..353d61ce 100644 --- a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java +++ b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.thrift.shaded.transport.TSocket; -import org.apache.thrift.shaded.transport.TTransport; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; import org.mockito.ArgumentMatchers; import javax.management.MalformedObjectNameException; diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java index abb2c5be..895af603 100644 --- a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java +++ b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java @@ -35,7 +35,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java index 4bef522c..7a43fc76 100644 --- a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java +++ b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.thrift.shaded.transport.TSocket; -import org.apache.thrift.shaded.transport.TTransport; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; import org.mockito.ArgumentMatchers; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java index a37c598f..b5df77a1 100644 --- a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java +++ b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java @@ -30,9 +30,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.thrift.shaded.TException; -import org.apache.thrift.shaded.transport.TSocket; -import org.apache.thrift.shaded.transport.TTransport; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; import org.mockito.ArgumentMatchers; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; diff --git a/rubix-build/pom.xml b/rubix-build/pom.xml new file mode 100644 index 00000000..22c4c1b4 --- /dev/null +++ b/rubix-build/pom.xml @@ -0,0 +1,116 @@ + + + + rubix-root + com.qubole.rubix + 0.3.17-SNAPSHOT + + 4.0.0 + + rubix-build + rubix-build + + ${project.parent.basedir} + + + + + com.qubole.rubix + rubix-spi + + + com.qubole.rubix + rubix-common + + + com.qubole.rubix + rubix-core + + + com.qubole.rubix + rubix-bookkeeper + + + com.qubole.rubix + rubix-hadoop2 + + + com.qubole.rubix + rubix-client + + + com.qubole.rubix + rubix-presto + + + com.qubole.rubix + rubix-prestosql + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + + + com.qubole.rubix:rubix-spi + com.qubole.rubix:rubix-common + com.qubole.rubix:rubix-core + com.qubole.rubix:rubix-bookkeeper + com.qubole.rubix:rubix-hadoop2 + com.qubole.rubix:rubix-prestosql + com.qubole.rubix:rubix-presto + com.qubole.rubix:rubix-client + + + + + org.apache.thrift + org.apache.thrift.shaded + + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + empty-javadoc-jar + package + + jar + + + javadoc + ${basedir}/javadoc + + + + + + + \ No newline at end of file diff --git a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java index 42de8439..f1b2750b 100644 --- a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java +++ b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java @@ -23,8 +23,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; -import org.apache.thrift.shaded.TException; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; import java.io.File; import java.io.IOException; diff --git a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java index 2abd78a9..e75e621a 100644 --- a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java +++ b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java @@ -13,7 +13,7 @@ package com.qubole.rubix.client.robotframework.container.client; import com.qubole.rubix.spi.RetryingPooledBookkeeperClient; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import java.io.Serializable; import java.util.HashMap; diff --git a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java index 0c16a76c..122d6585 100644 --- a/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java +++ b/rubix-client/src/test/java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java @@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.transport.TTransportException; import java.io.IOException; import java.net.URI; diff --git a/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java b/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java index 18af339a..787681e2 100644 --- a/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java +++ b/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java @@ -18,8 +18,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.thrift.shaded.TException; -import org.apache.thrift.shaded.transport.TTransportException; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; import org.testng.annotations.Test; import static org.mockito.Mockito.mock; diff --git a/rubix-presto-shaded/pom.xml b/rubix-presto-shaded/pom.xml index cfed8896..c41f4ef0 100644 --- a/rubix-presto-shaded/pom.xml +++ b/rubix-presto-shaded/pom.xml @@ -74,6 +74,10 @@ com.google.cloud.** + + org.apache.thrift + org.apache.thrift.shaded + diff --git a/rubix-spi/pom.xml b/rubix-spi/pom.xml index ee100b0d..ac225366 100644 --- a/rubix-spi/pom.xml +++ b/rubix-spi/pom.xml @@ -113,10 +113,6 @@ com.google com.google.shaded - - org.apache.thrift - org.apache.thrift.shaded - diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java index 50d94b66..b1b4f0aa 100644 --- a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java +++ b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java @@ -36,7 +36,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java index 4fcb09c6..dcfd04f4 100644 --- a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java +++ b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java @@ -41,7 +41,7 @@ import org.apache.hadoop.fs.Path; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.apache.thrift.shaded.TException; +import org.apache.thrift.TException; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass;