diff --git a/make-distribution.sh b/make-distribution.sh
index d46edbc50d152..55cbdc14ac055 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -181,6 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI
# Copy jars
cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/"
cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/"
+cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/"
# Copy example sources (needed for python and SQL)
mkdir -p "$DISTDIR/examples/src/main"
diff --git a/network/common/pom.xml b/network/common/pom.xml
index 8b24ebf1ba1f2..a6bee7ed09ad5 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -41,12 +41,13 @@
io.netty
netty-all
+
+
org.slf4j
slf4j-api
+ provided
-
-
com.google.guava
guava
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index a180a5e5f926e..be78331ea9b63 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -42,12 +42,13 @@
spark-network-common_${scala.binary.version}
${project.version}
+
+
org.slf4j
slf4j-api
+ provided
-
-
com.google.guava
guava
diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml
index 85960eb85b482..7845011ec3200 100644
--- a/network/yarn/pom.xml
+++ b/network/yarn/pom.xml
@@ -54,5 +54,38 @@
target/scala-${scala.binary.version}/classes
target/scala-${scala.binary.version}/test-classes
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+ false
+ ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar
+
+
+ *:*
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+ package
+
+ shade
+
+
+
+
+
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 5eb3ed439cde9..d95d50a1d8124 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -38,12 +38,12 @@ object BuildCommons {
"streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter",
"streaming-zeromq").map(ProjectRef(buildLocation, _))
- val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, networkYarn, java8Tests,
- sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "network-yarn",
+ val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests,
+ sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha",
"java8-tests", "ganglia-lgpl", "kinesis-asl").map(ProjectRef(buildLocation, _))
- val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples")
- .map(ProjectRef(buildLocation, _))
+ val assemblyProjects@Seq(assembly, examples, networkYarn) =
+ Seq("assembly", "examples", "network-yarn").map(ProjectRef(buildLocation, _))
val tools = ProjectRef(buildLocation, "tools")
// Root project.
@@ -289,8 +289,15 @@ object Assembly {
lazy val settings = assemblySettings ++ Seq(
test in assembly := {},
- jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" +
- Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" },
+ jarName in assembly <<= (version, moduleName) map { (v, mName) =>
+ if (mName.contains("network-yarn")) {
+ // This must match the same name used in maven (see network/yarn/pom.xml)
+ "spark-" + v + "-yarn-shuffle.jar"
+ } else {
+ mName + "-" + v + "-hadoop" +
+ Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar"
+ }
+ },
mergeStrategy in assembly := {
case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard