From b8895cb004232d8778e29f1f3e6605f69d9d93c0 Mon Sep 17 00:00:00 2001
From: Manuel Kiessling <manuel@kiessling.net>
Date: Wed, 18 Jan 2017 09:28:16 +0100
Subject: [PATCH]  #45: Upgrade to Spark 2.0.2 for stability

---
 Makefile                                      |   2 +-
 build.sbt                                     |  19 +--
 build/10-deploy-production                    |   7 +-
 spark/pom.xml                                 | 139 ++++++++++++++++++
 .../analyze/spark}/SparkApp.scala             |   0
 .../analyze/spark}/SparkAppSpec.scala         |   0
 6 files changed, 148 insertions(+), 19 deletions(-)
 create mode 100644 spark/pom.xml
 rename spark/src/main/scala/{ => com/journeymonitor/analyze/spark}/SparkApp.scala (100%)
 rename spark/src/test/scala/{ => com/journeymonitor/analyze/spark}/SparkAppSpec.scala (100%)

diff --git a/Makefile b/Makefile
index 6514abf..57ddf39 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ migrations:
 
 travisci-assembly-test:
 	sbt "project importer" "assembly"
-	sbt "project spark" "assembly"
+	cd spark && mvn package
 
 travisci-create-github-release:
 	[ "${TRAVIS_PULL_REQUEST}" = "false" ] && /bin/bash ./build/create-github-release.sh ${GITHUB_TOKEN} travisci-build-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER} ${TRAVIS_COMMIT} https://travis-ci.org/journeymonitor/analyze/builds/${TRAVIS_BUILD_ID}
diff --git a/build.sbt b/build.sbt
index 2dce568..8db65e5 100644
--- a/build.sbt
+++ b/build.sbt
@@ -30,25 +30,10 @@ lazy val json4sDependencies = Seq (
 )
 
 
-lazy val sparkDependencies = Seq (
-  "org.apache.spark" %% "spark-core" % "1.5.1" % "provided",
-  "com.datastax.spark" %% "spark-cassandra-connector" % "1.5.0-M2"
-)
-
-
 lazy val common = project
   .settings(commonSettings:_*)
   .settings(libraryDependencies ++= (testDependencies ++ cassandraDependencies))
 
-lazy val spark = project.in(file("spark"))
-  .settings(commonSettings:_*)
-  .settings(
-    javacOptions := Seq("-source", "1.7", "-target", "1.7"),
-    scalacOptions := Seq("-target:jvm-1.7", "-unchecked", "-deprecation", "-encoding", "utf8")
-  )
-  .settings(libraryDependencies ++= (sparkDependencies ++ json4sDependencies ++ testDependencies))
-  .settings(assemblyJarName in assembly := "journeymonitor-analyze-spark-assembly.jar")
-
 lazy val importer = project.in(file("importer"))
   .settings(commonSettings:_*)
   .settings(libraryDependencies ++= (json4sDependencies ++ testDependencies ++ cassandraDependencies))
@@ -61,4 +46,6 @@ lazy val api = project
   .dependsOn(common)
 
 lazy val main = project.in(file("."))
-  .aggregate(common, spark, importer, api)
+  .aggregate(common, importer, api)
+
+// "spark" is not an sbt sub-project, it's a standalone Maven project - see spark/pom.xml
diff --git a/build/10-deploy-production b/build/10-deploy-production
index 0d82960..79071d0 100755
--- a/build/10-deploy-production
+++ b/build/10-deploy-production
@@ -15,11 +15,14 @@ cd $2
 /usr/bin/sbt "project common" "run" || exit 1 # This applies Pillar migrations for test and prod env
 /usr/bin/sbt "project common" "test" || exit 1
 /usr/bin/sbt "project importer" "assembly" || exit 1 # This also runs all tests
-/usr/bin/sbt "project spark" "assembly" || exit 1 # This also runs all tests
 /usr/bin/sbt "project api" "test" "universal:packageZipTarball" || exit 1
 
+cd spark
+/usr/bin/mvn package || exit 1 # This also runs all tests
+cd -
+
 cp importer/target/scala-2.11/journeymonitor-analyze-importer-assembly.jar /opt/journeymonitor/analyze/
-cp spark/target/scala-2.11/journeymonitor-analyze-spark-assembly.jar /opt/journeymonitor/analyze/
+cp spark/target/spark-1.0-SNAPSHOT-jar-with-dependencies.jar /opt/journeymonitor/analyze/
 
 cp api/target/universal/api-1.0-SNAPSHOT.tgz /opt/journeymonitor/analyze/
 /bin/kill `cat /var/tmp/journeymonitor-analyze-api.pid`
diff --git a/spark/pom.xml b/spark/pom.xml
new file mode 100644
index 0000000..ad9cb5c
--- /dev/null
+++ b/spark/pom.xml
@@ -0,0 +1,139 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.journeymonitor.analyze</groupId>
+  <artifactId>spark</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>${project.artifactId}</name>
+  <description>My wonderfull scala app</description>
+  <inceptionYear>2015</inceptionYear>
+  <licenses>
+    <license>
+      <name>My License</name>
+      <url>http://....</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+
+  <properties>
+    <encoding>UTF-8</encoding>
+    <java.version>1.8</java.version>
+    <scala.version>2.11.8</scala.version>
+    <scala.compat.version>2.11</scala.compat.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scala.compat.version}</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.compat.version}</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.datastax.spark</groupId>
+      <artifactId>spark-cassandra-connector_${scala.compat.version}</artifactId>
+      <version>2.0.0-M3</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.json4s</groupId>
+      <artifactId>json4s-jackson_${scala.compat.version}</artifactId>
+      <version>3.5.0</version>
+    </dependency>
+
+    <!-- Test -->
+    <dependency>
+      <groupId>org.scalatest</groupId>
+      <artifactId>scalatest_${scala.compat.version}</artifactId>
+      <version>3.0.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <sourceDirectory>src/main/scala</sourceDirectory>
+    <testSourceDirectory>src/test/scala</testSourceDirectory>
+    <plugins>
+
+      <plugin>
+        <groupId>net.alchim31.maven</groupId>
+        <artifactId>scala-maven-plugin</artifactId>
+        <version>3.2.2</version>
+        <executions>
+          <execution>
+            <id>scala-compile</id>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>add-source</goal>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>scala-test-compile</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <args>
+            <arg>-feature</arg>
+          </args>
+          <recompileMode>incremental</recompileMode>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.scalatest</groupId>
+        <artifactId>scalatest-maven-plugin</artifactId>
+        <version>1.0</version>
+        <executions>
+          <execution>
+            <id>test</id>
+            <goals>
+              <goal>test</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <!-- "package" command plugin -->
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.6</version>
+        <configuration>
+          <archive>
+            <manifest>
+              <mainClass>com.journeymonitor.analyze.spark.SparkApp</mainClass>
+            </manifest>
+          </archive>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+    </plugins>
+  </build>
+</project>
diff --git a/spark/src/main/scala/SparkApp.scala b/spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala
similarity index 100%
rename from spark/src/main/scala/SparkApp.scala
rename to spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala
diff --git a/spark/src/test/scala/SparkAppSpec.scala b/spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala
similarity index 100%
rename from spark/src/test/scala/SparkAppSpec.scala
rename to spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala