#45: Upgrade to Spark 2.0.2 for stability

Makefile

@@ -9,7 +9,7 @@ migrations:
 
 travisci-assembly-test:
 	sbt "project importer" "assembly"
-	sbt "project spark" "assembly"
+	cd spark && mvn package
 
 travisci-create-github-release:
 	[ "${TRAVIS_PULL_REQUEST}" = "false" ] && /bin/bash ./build/create-github-release.sh ${GITHUB_TOKEN} travisci-build-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER} ${TRAVIS_COMMIT} https://travis-ci.org/journeymonitor/analyze/builds/${TRAVIS_BUILD_ID}

build.sbt

@@ -30,25 +30,10 @@ lazy val json4sDependencies = Seq (
 )
 
 
-lazy val sparkDependencies = Seq (
-  "org.apache.spark" %% "spark-core" % "1.5.1" % "provided",
-  "com.datastax.spark" %% "spark-cassandra-connector" % "1.5.0-M2"
-)
-
-
 lazy val common = project
   .settings(commonSettings:_*)
   .settings(libraryDependencies ++= (testDependencies ++ cassandraDependencies))
 
-lazy val spark = project.in(file("spark"))
-  .settings(commonSettings:_*)
-  .settings(
-    javacOptions := Seq("-source", "1.7", "-target", "1.7"),
-    scalacOptions := Seq("-target:jvm-1.7", "-unchecked", "-deprecation", "-encoding", "utf8")
-  )
-  .settings(libraryDependencies ++= (sparkDependencies ++ json4sDependencies ++ testDependencies))
-  .settings(assemblyJarName in assembly := "journeymonitor-analyze-spark-assembly.jar")
-
 lazy val importer = project.in(file("importer"))
   .settings(commonSettings:_*)
   .settings(libraryDependencies ++= (json4sDependencies ++ testDependencies ++ cassandraDependencies))
@@ -61,4 +46,6 @@ lazy val api = project
   .dependsOn(common)
 
 lazy val main = project.in(file("."))
-  .aggregate(common, spark, importer, api)
+  .aggregate(common, importer, api)
+
+// "spark" is not an sbt sub-project, it's a standalone Maven project - see spark/pom.xml

build/10-deploy-production

@@ -15,11 +15,14 @@ cd $2
 /usr/bin/sbt "project common" "run" || exit 1 # This applies Pillar migrations for test and prod env
 /usr/bin/sbt "project common" "test" || exit 1
 /usr/bin/sbt "project importer" "assembly" || exit 1 # This also runs all tests
-/usr/bin/sbt "project spark" "assembly" || exit 1 # This also runs all tests
 /usr/bin/sbt "project api" "test" "universal:packageZipTarball" || exit 1
 
+cd spark
+/usr/bin/mvn package || exit 1 # This also runs all tests
+cd -
+
 cp importer/target/scala-2.11/journeymonitor-analyze-importer-assembly.jar /opt/journeymonitor/analyze/
-cp spark/target/scala-2.11/journeymonitor-analyze-spark-assembly.jar /opt/journeymonitor/analyze/
+cp spark/target/spark-1.0-SNAPSHOT-jar-with-dependencies.jar /opt/journeymonitor/analyze/
 
 cp api/target/universal/api-1.0-SNAPSHOT.tgz /opt/journeymonitor/analyze/
 /bin/kill `cat /var/tmp/journeymonitor-analyze-api.pid`

spark/pom.xml

@@ -0,0 +1,139 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.journeymonitor.analyze</groupId>
+  <artifactId>spark</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <name>${project.artifactId}</name>
+  <description>My wonderfull scala app</description>
+  <inceptionYear>2015</inceptionYear>
+  <licenses>
+    <license>
+      <name>My License</name>
+      <url>http://....</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+
+  <properties>
+    <encoding>UTF-8</encoding>
+    <java.version>1.8</java.version>
+    <scala.version>2.11.8</scala.version>
+    <scala.compat.version>2.11</scala.compat.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scala.compat.version}</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.compat.version}</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.datastax.spark</groupId>
+      <artifactId>spark-cassandra-connector_${scala.compat.version}</artifactId>
+      <version>2.0.0-M3</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.json4s</groupId>
+      <artifactId>json4s-jackson_${scala.compat.version}</artifactId>
+      <version>3.5.0</version>
+    </dependency>
+
+    <!-- Test -->
+    <dependency>
+      <groupId>org.scalatest</groupId>
+      <artifactId>scalatest_${scala.compat.version}</artifactId>
+      <version>3.0.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <sourceDirectory>src/main/scala</sourceDirectory>
+    <testSourceDirectory>src/test/scala</testSourceDirectory>
+    <plugins>
+
+      <plugin>
+        <groupId>net.alchim31.maven</groupId>
+        <artifactId>scala-maven-plugin</artifactId>
+        <version>3.2.2</version>
+        <executions>
+          <execution>
+            <id>scala-compile</id>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>add-source</goal>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+          <execution<