Remove shading of thrift lib from submodule (#446)

Shade thrift and all submodules into single jar in rubix-build module Co-authored-by: Suryansh Agnihotri <suryansha@qubole.com>
qubole · Sep 4, 2020 · acd7608 · acd7608
1 parent 266499c
commit acd7608
Show file tree

Hide file tree

Showing 20 changed files with 150 additions and 32 deletions.
diff --git a/docs/release/release_notes/release-0.3.17-(next-release).rst b/docs/release/release_notes/release-0.3.17-(next-release).rst
@@ -4,4 +4,5 @@ Release 0.3.17 (next release)
 
 Fixes and Features
 ------------------
-* Added `total_system_source_mb_read` stat in detailed metrics to show total data read from source: during read + warmups
+* Added `total_system_source_mb_read` stat in detailed metrics to show total data read from source: during read + warmups
+* Moved shading of Thrift classes into a common sub-module rubix-build. Project is now traversable in IDE and mvn test works at the root. Clients should now include rubix-build artifact instead of including sub-modules independently
diff --git a/pom.xml b/pom.xml
@@ -70,6 +70,7 @@
         <module>assembly</module>
         <module>rubix-rpm</module>
         <module>rubix-presto-shaded</module>
+        <module>rubix-build</module>
     </modules>
 
     <repositories>

diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeper.java
@@ -56,7 +56,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.DirectBufferPool;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 
 import java.io.File;
 import java.io.FileNotFoundException;

diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/BookKeeperServer.java
@@ -27,11 +27,11 @@
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.thrift.shaded.server.TServer;
-import org.apache.thrift.shaded.server.TThreadPoolServer;
-import org.apache.thrift.shaded.transport.TServerSocket;
-import org.apache.thrift.shaded.transport.TServerTransport;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransportException;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;

diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/HeartbeatService.java
@@ -28,7 +28,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 
 import java.io.IOException;
 import java.net.InetAddress;

diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/LocalDataTransferServer.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;

diff --git a/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java b/rubix-bookkeeper/src/main/java/com/qubole/rubix/bookkeeper/validation/CachingValidator.java
@@ -27,7 +27,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/BaseServerTest.java
@@ -22,9 +22,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.shaded.transport.TSocket;
-import org.apache.thrift.shaded.transport.TTransport;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
 import org.mockito.ArgumentMatchers;
 
 import javax.management.MalformedObjectNameException;

diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestBookKeeper.java
@@ -35,7 +35,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;

diff --git a/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java b/rubix-bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/TestHeartbeatService.java
@@ -22,9 +22,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.shaded.transport.TSocket;
-import org.apache.thrift.shaded.transport.TTransport;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
 import org.mockito.ArgumentMatchers;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;

diff --git a/...bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java b/...bookkeeper/src/test/java/com/qubole/rubix/bookkeeper/validation/TestCachingValidator.java
@@ -30,9 +30,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.shaded.TException;
-import org.apache.thrift.shaded.transport.TSocket;
-import org.apache.thrift.shaded.transport.TTransport;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
 import org.mockito.ArgumentMatchers;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;

diff --git a/rubix-build/pom.xml b/rubix-build/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>rubix-root</artifactId>
+        <groupId>com.qubole.rubix</groupId>
+        <version>0.3.17-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>rubix-build</artifactId>
+    <name>rubix-build</name>
+    <properties>
+        <main.basedir>${project.parent.basedir}</main.basedir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-spi</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-bookkeeper</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-hadoop2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-presto</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qubole.rubix</groupId>
+            <artifactId>rubix-prestosql</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.3</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <includes>
+                                    <include>com.qubole.rubix:rubix-spi</include>
+                                    <include>com.qubole.rubix:rubix-common</include>
+                                    <include>com.qubole.rubix:rubix-core</include>
+                                    <include>com.qubole.rubix:rubix-bookkeeper</include>
+                                    <include>com.qubole.rubix:rubix-hadoop2</include>
+                                    <include>com.qubole.rubix:rubix-prestosql</include>
+                                    <include>com.qubole.rubix:rubix-presto</include>
+                                    <include>com.qubole.rubix:rubix-client</include>
+                                </includes>
+                            </artifactSet>
+                            <relocations>
+                                <relocation>
+                                    <pattern>org.apache.thrift</pattern>
+                                    <shadedPattern>org.apache.thrift.shaded</shadedPattern>
+                                </relocation>
+                            </relocations>
+
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <!-- javadoc is a compulsory requirement of sonatype, create empty one for this module -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>empty-javadoc-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                        <configuration>
+                            <classifier>javadoc</classifier>
+                            <classesDirectory>${basedir}/javadoc</classesDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/...lient/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java b/...lient/src/test/java/com/qubole/rubix/client/robotframework/BookKeeperClientRFLibrary.java
@@ -23,8 +23,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.thrift.shaded.TException;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/.../java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java b/.../java/com/qubole/rubix/client/robotframework/container/client/GetCacheMetricsRequest.java
@@ -13,7 +13,7 @@
 package com.qubole.rubix.client.robotframework.container.client;
 
 import com.qubole.rubix.spi.RetryingPooledBookkeeperClient;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 
 import java.io.Serializable;
 import java.util.HashMap;

diff --git a/.../java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java b/.../java/com/qubole/rubix/client/robotframework/container/server/ContainerRequestServer.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.transport.TTransportException;
 
 import java.io.IOException;
 import java.net.URI;

diff --git a/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java b/rubix-client/src/test/java/com/qubole/rubix/health/TestBookKeeperHealth.java
@@ -18,8 +18,8 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.thrift.shaded.TException;
-import org.apache.thrift.shaded.transport.TTransportException;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
 import org.testng.annotations.Test;
 
 import static org.mockito.Mockito.mock;

diff --git a/rubix-presto-shaded/pom.xml b/rubix-presto-shaded/pom.xml
@@ -74,6 +74,10 @@
                                         <exclude>com.google.cloud.**</exclude>
                                     </excludes>
                                 </relocation>
+                                <relocation>
+                                    <pattern>org.apache.thrift</pattern>
+                                    <shadedPattern>org.apache.thrift.shaded</shadedPattern>
+                                </relocation>
                             </relocations>
 
                         </configuration>

diff --git a/rubix-spi/pom.xml b/rubix-spi/pom.xml
@@ -113,10 +113,6 @@
                                     <pattern>com.google</pattern>
                                     <shadedPattern>com.google.shaded</shadedPattern>
                                 </relocation>
-                               <relocation>
-                                    <pattern>org.apache.thrift</pattern>
-                                    <shadedPattern>org.apache.thrift.shaded</shadedPattern>
-                                </relocation>
                             </relocations>
                         </configuration>
                     </execution>

diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestGenerationNumber.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;

diff --git a/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java b/rubix-tests/src/test/java/com/qubole/rubix/tests/TestNonLocalReadRequestChain.java
@@ -41,7 +41,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
-import org.apache.thrift.shaded.TException;
+import org.apache.thrift.TException;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;