From 67f3d8753b03758d89a67f4283796f7e4f3db002 Mon Sep 17 00:00:00 2001 From: Darren Fuller Date: Sun, 21 Feb 2021 10:47:23 +0000 Subject: [PATCH 1/2] add config for Spark 3.0.2, and produce script to build for different spark versions --- build.ps1 | 17 +++++++++++++++++ build.sbt | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 build.ps1 diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 0000000..aa09063 --- /dev/null +++ b/build.ps1 @@ -0,0 +1,17 @@ +$versions = @("3.0.1", "3.0.2") +$jarPath = "./target/jars" + +Write-Host "Clearing existing jar artefacts" -ForegroundColor Green +if (Test-Path $jarPath) { + Remove-Item -Path $jarPath -Force -Recurse +} + +New-Item -Path $jarPath -ItemType Directory + +foreach ($version in $versions) { + Write-Host "Building for Spark version: $version" -ForegroundColor Green + & sbt -DsparkVersion="$version" clean compile test package +} + +Write-Host "Copying jar files to $jarPath" -ForegroundColor Green +Get-ChildItem -Filter "spark-cef*.jar" -Path ./target -Recurse | Copy-Item -Destination $jarPath diff --git a/build.sbt b/build.sbt index 2d1c1ce..39c226b 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ val sparkVersion = settingKey[String]("Spark version") val scalaTestVersion = settingKey[String]("ScalaTest version") name := "spark-cef-reader" -version := "0.1-SNAPSHOT" +version := "0.2-SNAPSHOT" organization := "com.bp" description := "CEF data source for Spark" homepage := Some(url("https://github.com/bp")) @@ -14,6 +14,8 @@ developers ++= List( Developer(id = "azurecoder", name = "Richard Conway", email = "richard@elastacloud.com", url = url("https://github.com/elastacloud")) ) +target := file("target") / s"spark-${sparkVersion.value}" + artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) => s"${artifact.name}-${sv.binary}_${sparkVersion.value}-${module.revision}.${artifact.extension}" } @@ -34,7 +36,7 @@ libraryDependencies ++= Seq( // Define common settings for the library val commonSettings = Seq( - sparkVersion := System.getProperty("sparkVersion", "3.0.1"), + sparkVersion := System.getProperty("sparkVersion", "3.0.2"), scalaVersion := { if (sparkVersion.value < "3.0.0") { "2.11.12" From 7da3761be29cc0dfea2a82d2a120b304dd8eec24 Mon Sep 17 00:00:00 2001 From: Darren Fuller Date: Wed, 3 Mar 2021 11:03:13 +0000 Subject: [PATCH 2/2] as it arrived so quickly, added Spark 3.1.1 --- build.ps1 | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.ps1 b/build.ps1 index aa09063..07b9f8d 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,4 +1,4 @@ -$versions = @("3.0.1", "3.0.2") +$versions = @("3.0.1", "3.0.2", "3.1.1") $jarPath = "./target/jars" Write-Host "Clearing existing jar artefacts" -ForegroundColor Green diff --git a/build.sbt b/build.sbt index 39c226b..be03caf 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ libraryDependencies ++= Seq( // Define common settings for the library val commonSettings = Seq( - sparkVersion := System.getProperty("sparkVersion", "3.0.2"), + sparkVersion := System.getProperty("sparkVersion", "3.1.1"), scalaVersion := { if (sparkVersion.value < "3.0.0") { "2.11.12"