From 24948136d028ed5289779eea27ed93861e735d34 Mon Sep 17 00:00:00 2001
From: Scott Sandre <scott.sandre@databricks.com>
Date: Wed, 1 May 2024 14:16:34 -0700
Subject: [PATCH] done

---
 .github/workflows/spark_examples_test.yaml | 52 ++++++++++++++++++++++
 .github/workflows/spark_test.yaml          |  1 -
 examples/scala/build.sbt                   | 23 +++++++---
 3 files changed, 68 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/spark_examples_test.yaml

diff --git a/.github/workflows/spark_examples_test.yaml b/.github/workflows/spark_examples_test.yaml
new file mode 100644
index 00000000000..11c676b573b
--- /dev/null
+++ b/.github/workflows/spark_examples_test.yaml
@@ -0,0 +1,52 @@
+name: "Delta Spark Local Publishing and Examples Compilation"
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        # These Scala versions must match those in the build.sbt
+        scala: [2.12.18, 2.13.13]
+    env:
+      SCALA_VERSION: ${{ matrix.scala }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v4
+        id: git-diff
+        with:
+          PATTERNS: |
+            **
+            .github/workflows/**
+            !kernel/**
+            !connectors/**
+      - name: install java
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "8"
+      - name: Cache Scala, SBT
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2
+            ~/.cache/coursier
+          # Change the key if dependencies are changed. For each key, GitHub Actions will cache the
+          # the above directories when we use the key for the first time. After that, each run will
+          # just use the cache. The cache is immutable so we need to use a new key when trying to
+          # cache new stuff.
+          key: delta-sbt-cache-spark-examples-scala${{ matrix.scala }}
+      - name: Install Job dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
+          sudo apt install libedit-dev
+        if: steps.git-diff.outputs.diff
+      - name: Run Delta Spark Local Publishing and Examples Compilation
+        # examples/scala/build.sbt will compile against the local Delta relase version (e.g. 3.2.0-SNAPSHOT).
+        # Thus, we need to publishM2 first so those jars are locally accessible.
+        run: |
+          build/sbt clean
+          build/sbt "++ $SCALA_VERSION publishM2"
+          cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
+        if: steps.git-diff.outputs.diff
diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml
index 85d30f15f24..5f533e2ca98 100644
--- a/.github/workflows/spark_test.yaml
+++ b/.github/workflows/spark_test.yaml
@@ -67,5 +67,4 @@ jobs:
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_master_test.yaml
         run: |
           TEST_PARALLELISM_COUNT=2 pipenv run python run-tests.py --group spark
-          cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
         if: steps.git-diff.outputs.diff
diff --git a/examples/scala/build.sbt b/examples/scala/build.sbt
index a1f88537823..a6fe73eb0cd 100644
--- a/examples/scala/build.sbt
+++ b/examples/scala/build.sbt
@@ -20,8 +20,15 @@ organizationName := "example"
 
 val scala212 = "2.12.18"
 val scala213 = "2.13.13"
-val deltaVersion = "3.0.0"
 val icebergVersion = "1.4.1"
+val defaultDeltaVersion = {
+  val versionFileContent = IO.read(file("../../version.sbt"))
+  val versionRegex = """.*version\s*:=\s*"([^"]+)".*""".r
+  versionRegex.findFirstMatchIn(versionFileContent) match {
+    case Some(m) => m.group(1)
+    case None => throw new Exception("Could not parse version from version.sbt")
+  }
+}
 
 def getMajorMinor(version: String): (Int, Int) = {
   val majorMinor = Try {
@@ -58,7 +65,7 @@ val getScalaVersion = settingKey[String](
   s"get scala version from environment variable SCALA_VERSION. If it doesn't exist, use $scala213"
 )
 val getDeltaVersion = settingKey[String](
-  s"get delta version from environment variable DELTA_VERSION. If it doesn't exist, use $deltaVersion"
+  s"get delta version from environment variable DELTA_VERSION. If it doesn't exist, use $defaultDeltaVersion"
 )
 val getDeltaArtifactName = settingKey[String](
   s"get delta artifact name based on the delta version. either `delta-core` or `delta-spark`."
@@ -68,13 +75,14 @@ val getIcebergSparkRuntimeArtifactName = settingKey[String](
 )
 getScalaVersion := {
   sys.env.get("SCALA_VERSION") match {
-    case Some("2.12") =>
+    case Some("2.12") | Some(`scala212`) =>
       scala212
-    case Some("2.13") =>
+    case Some("2.13") | Some(`scala213`) =>
       scala213
     case Some(v) =>
       println(
-        s"[warn] Invalid  SCALA_VERSION. Expected 2.12 or 2.13 but got $v. Fallback to $scala213."
+        s"[warn] Invalid  SCALA_VERSION. Expected one of {2.12, $scala212, 2.13, $scala213} but " +
+        s"got $v. Fallback to $scala213."
       )
       scala213
     case None =>
@@ -88,10 +96,11 @@ version := "0.1.0"
 getDeltaVersion := {
   sys.env.get("DELTA_VERSION") match {
     case Some(v) =>
-      println(s"Using Delta version $v")
+      println(s"Using DELTA_VERSION Delta version $v")
       v
     case None =>
-      deltaVersion
+      println(s"Using default Delta version $defaultDeltaVersion")
+      defaultDeltaVersion
   }
 }