use hadoop 2.8.0-palantir2 (apache#107)

mccheah · Feb 27, 2017 · a22ccff · a22ccff
1 parent 8559496
commit a22ccff
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 23 deletions.
diff --git a/circle.yml b/circle.yml
@@ -22,7 +22,7 @@ dependencies:
     - PYENV_VERSION=3.4.4 pip install numpy
     #- PYENV_VERSION=pypy-4.0.1 pip install numpy
   override:
-    - ./build/mvn -DskipTests -Phadoop-2.7 -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr dependency:go-offline
+    - ./build/mvn -DskipTests -Phadoop-palantir -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr dependency:go-offline
 
 general:
   artifacts:
@@ -31,9 +31,9 @@ general:
 
 test:
   override:
-    - set -o pipefail && HADOOP_PROFILE=hadoop2.7 ./dev/run-tests | tee -a "$CIRCLE_ARTIFACTS/run-tests.log" | grep -v -i info:
+    - set -o pipefail && HADOOP_PROFILE=hadooppalantir ./dev/run-tests | tee -a "$CIRCLE_ARTIFACTS/run-tests.log" | grep -v -i info:
         parallel: true
-        timeout: 1200
+        timeout: 1800
         max-runtime: 14400
     - ? |
           set -euo pipefail
@@ -47,7 +47,7 @@ test:
             echo "</server></servers></settings>" >> $tmp_settings
 
             ./build/mvn versions:set -DnewVersion=$version
-            ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr clean deploy
+            ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-palantir -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr clean deploy
           }
 
           make_dist() {
@@ -64,7 +64,7 @@ test:
             publish_artifacts
             ;;
           1)
-            make_dist hadoop-2.7 "-Phadoop-2.7 -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr"
+            make_dist hadoop-2.8.0-palantir2 "-Phadoop-palantir -Pkinesis-asl -Pmesos -Pyarn -Phive-thriftserver -Phive -Psparkr"
             ;;
           2)
             make_dist without-hadoop "-Phadoop-provided -Pmesos -Pyarn -Psparkr"

diff --git a/core/pom.xml b/core/pom.xml
@@ -58,6 +58,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-launcher_${scala.binary.version}</artifactId>

diff --git a/dev/deps/spark-deps-hadoop-2.7 → dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-2.7 → dev/deps/spark-deps-hadoop-palantir
@@ -60,25 +60,27 @@ gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
 guice-servlet-3.0.jar
-hadoop-annotations-2.7.3.jar
-hadoop-auth-2.7.3.jar
-hadoop-client-2.7.3.jar
-hadoop-common-2.7.3.jar
-hadoop-hdfs-2.7.3.jar
-hadoop-mapreduce-client-app-2.7.3.jar
-hadoop-mapreduce-client-common-2.7.3.jar
-hadoop-mapreduce-client-core-2.7.3.jar
-hadoop-mapreduce-client-jobclient-2.7.3.jar
-hadoop-mapreduce-client-shuffle-2.7.3.jar
-hadoop-yarn-api-2.7.3.jar
-hadoop-yarn-client-2.7.3.jar
-hadoop-yarn-common-2.7.3.jar
-hadoop-yarn-server-common-2.7.3.jar
-hadoop-yarn-server-web-proxy-2.7.3.jar
+hadoop-annotations-2.8.0-palantir2.jar
+hadoop-auth-2.8.0-palantir2.jar
+hadoop-client-2.8.0-palantir2.jar
+hadoop-common-2.8.0-palantir2.jar
+hadoop-hdfs-2.8.0-palantir2.jar
+hadoop-hdfs-client-2.8.0-palantir2.jar
+hadoop-mapreduce-client-app-2.8.0-palantir2.jar
+hadoop-mapreduce-client-common-2.8.0-palantir2.jar
+hadoop-mapreduce-client-core-2.8.0-palantir2.jar
+hadoop-mapreduce-client-jobclient-2.8.0-palantir2.jar
+hadoop-mapreduce-client-shuffle-2.8.0-palantir2.jar
+hadoop-yarn-api-2.8.0-palantir2.jar
+hadoop-yarn-client-2.8.0-palantir2.jar
+hadoop-yarn-common-2.8.0-palantir2.jar
+hadoop-yarn-server-common-2.8.0-palantir2.jar
+hadoop-yarn-server-web-proxy-2.8.0-palantir2.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
-htrace-core-3.1.0-incubating.jar
+hpack-0.11.0.jar
+htrace-core4-4.0.1-incubating.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
@@ -101,6 +103,7 @@ javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
 javolution-5.5.1.jar
 jaxb-api-2.2.2.jar
+jcip-annotations-1.0.jar
 jcl-over-slf4j-1.7.16.jar
 jdo-api-3.0.1.jar
 jersey-client-2.22.2.jar
@@ -112,11 +115,13 @@ jersey-media-jaxb-2.22.2.jar
 jersey-server-2.22.2.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
+jetty-sslengine-6.1.26.jar
 jetty-util-6.1.26.jar
 jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
+json-smart-1.1.1.jar
 json4s-ast_2.11-3.2.11.jar
 json4s-core_2.11-3.2.11.jar
 json4s-jackson_2.11-3.2.11.jar
@@ -141,7 +146,10 @@ minlog-1.3.0.jar
 mx4j-3.0.2.jar
 netty-3.9.9.Final.jar
 netty-all-4.0.43.Final.jar
+nimbus-jose-jwt-3.9.jar
 objenesis-2.1.jar
+okhttp-2.4.0.jar
+okio-1.4.0.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar

diff --git a/dev/run-tests.py b/dev/run-tests.py
@@ -310,6 +310,7 @@ def get_hadoop_profiles(hadoop_version):
         "hadoop2.4": ["-Phadoop-2.4"],
         "hadoop2.6": ["-Phadoop-2.6"],
         "hadoop2.7": ["-Phadoop-2.7"],
+        "hadooppalantir": ["-Phadoop-palantir"],
     }
 
     if hadoop_version in sbt_maven_hadoop_profiles:

diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
@@ -32,7 +32,7 @@ export LC_ALL=C
 HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pyarn -Phive"
 MVN="build/mvn"
 HADOOP_PROFILES=(
-    hadoop-2.7
+    hadoop-palantir
 )
 
 # We'll switch the version to a temp. one, publish POMs using that new version, then switch back to

diff --git a/pom.xml b/pom.xml
@@ -880,6 +880,79 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <!-- TODO (srand) Remove this when https://issues.apache.org/jira/browse/HDFS-11431 is fixed -->
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <version>${hadoop.version}</version>
+        <scope>${hadoop.deps.scope}</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-mapper-asl</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-daemon</groupId>
+            <artifactId>commons-daemon</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-net</groupId>
+            <artifactId>commons-net</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>servlet-api-2.5</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey.jersey-test-framework</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey.contribs</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro</artifactId>
@@ -2599,6 +2672,16 @@
       </properties>
     </profile>
 
+    <profile>
+      <id>hadoop-palantir</id>
+      <properties>
+	<hadoop.version>2.8.0-palantir2</hadoop.version>
+	<jets3t.version>0.9.3</jets3t.version>
+	<zookeeper.version>3.4.6</zookeeper.version>
+	<curator.version>2.6.0</curator.version>
+      </properties>
+    </profile>
+
     <profile>
       <id>yarn</id>
       <modules>

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -113,7 +113,8 @@ private[hive] object IsolatedClientLoader extends Logging {
       SparkSubmitUtils.resolveMavenCoordinates(
         hiveArtifacts.mkString(","),
         SparkSubmitUtils.buildIvySettings(
-          Some("http://www.datanucleus.org/downloads/maven2"),
+          Some("http://www.datanucleus.org/downloads/maven2," +
+            "http://dl.bintray.com/palantir/releases"),
           ivyPath),
         exclusions = version.exclusions)
     }