Prepackage deploy step (apache#331)

mccheah · Mar 23, 2018 · 1372eb2 · 1372eb2
1 parent fc092cb
commit 1372eb2
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 34 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,6 +21,13 @@ all-branches-and-tags: &all-branches-and-tags
     tags:
       only: /.*/
 
+deployable-branches-and-tags: &deployable-branches-and-tags
+  filters:
+    tags:
+      only: /[0-9]+(?:\.[0-9]+){2,}-palantir\.[0-9]+(?:\.[0-9]+)*/
+    branches:
+      only: master
+
 
 jobs:
   build-maven:
@@ -323,28 +330,50 @@ jobs:
       - store_artifacts:
           path: /tmp/yarn-tests
 
-  deploy:
+  build-maven-versioned:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
       - image: palantirtechnologies/circle-spark-r
     steps:
       - checkout
-      - attach_workspace:
-          at: .
       - restore_cache:
           key: maven-dependency-cache-{{ checksum "pom.xml" }}
       - restore_cache:
           key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
+      - run:
+          command: dev/set_version_and_package.sh
+      # This is potentially costly but we can't use the workspace as it would conflict with
+      # compilation results from build-sbt
+      - save_cache:
+          key: v1-maven-build-with-version-{{ .Branch }}-{{ .BuildNum }}
+          paths:
+            - .
+
+  deploy:
+    <<: *defaults
+    # Some part of the maven setup fails if there's no R, so we need to use the R image here
+    docker:
+      - image: palantirtechnologies/circle-spark-r
+    steps:
+      # This cache contains the whole project after version was set and mvn package was called
+      # Restoring first (and instead of checkout) as mvn versions:set mutates real source code...
+      - restore_cache:
+          key: v1-maven-build-with-version-{{ .Branch }}-{{ .BuildNum }}
+      - restore_cache:
+          key: maven-dependency-cache-{{ checksum "pom.xml" }}
+      - restore_cache:
+          key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
+
       - run: echo "user=$BINTRAY_USERNAME" > .credentials
       - run: echo "password=$BINTRAY_PASSWORD" >> .credentials
       - run: echo "realm=Bintray API Realm" >> .credentials
       - run: echo "host=api.bintray.com" >> .credentials
       - deploy:
           command: dev/publish.sh
       - store_artifacts:
-          path: /tmp/make-distribution.log
-          destination: make-distribution.log
+          path: /tmp/make-dist.log
+          destination: make-dist.log
       - store_artifacts:
           path: /tmp/publish_artifacts.log
           destination: publish_artifacts.log
@@ -383,6 +412,10 @@ workflows:
           requires:
             - build-sbt
           <<: *all-branches-and-tags
+      - build-maven-versioned:
+          requires:
+            - build-maven
+          <<: *deployable-branches-and-tags
       - deploy:
           requires:
             - build-maven
@@ -393,8 +426,5 @@ workflows:
             - run-scala-tests
             - run-python-tests
             - run-r-tests
-          filters:
-            tags:
-              only: /[0-9]+(?:\.[0-9]+){2,}-palantir\.[0-9]+(?:\.[0-9]+)*/
-            branches:
-              only: master
+            - build-maven-versioned
+          <<: *deployable-branches-and-tags
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
@@ -107,6 +107,8 @@ circle.yml
 .credentials
 publish.sh
 publish-local.sh
+publish_functions.sh
+set_version_and_package.sh
 structured-streaming/*
 kafka-source-initial-offset-version-2.1.0.bin
 kafka-source-initial-offset-future-version.bin
diff --git a/dev/publish.sh b/dev/publish.sh
@@ -1,31 +1,10 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-version=$(git describe --tags --first-parent)
 
-PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
+FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 
-publish_artifacts() {
-  tmp_settings="tmp-settings.xml"
-  echo "<settings><servers><server>" > $tmp_settings
-  echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
-  echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
-  echo "</server></servers></settings>" >> $tmp_settings
-
-  ./build/mvn versions:set -DnewVersion=$version
-  ./build/mvn --settings $tmp_settings -DskipTests "${PALANTIR_FLAGS[@]}" deploy
-}
-
-make_dist() {
-  build_flags="$1"
-  shift 1
-  hadoop_name="hadoop-palantir"
-  artifact_name="spark-dist_2.11-${hadoop_name}"
-  file_name="spark-dist-${version}-${hadoop_name}.tgz"
-  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags | tee -a "/tmp/make-distribution.log"
-  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz"
-  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST "https://api.bintray.com/content/palantir/releases/spark/${version}/publish"
-}
+source "$FWDIR/publish_functions.sh"
 
 publish_artifacts | tee -a "/tmp/publish_artifacts.log"
-make_dist "${PALANTIR_FLAGS[*]}" --clean
+make_dist "${PALANTIR_FLAGS[*]}" | tee -a "/tmp/make-dist.log"
diff --git a/dev/publish_functions.sh b/dev/publish_functions.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
+
+get_version() {
+  git describe --tags --first-parent
+}
+
+set_version_and_package() {
+  version=$(get_version)
+  ./build/mvn versions:set -DnewVersion="$version"
+  ./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" package
+}
+
+publish_artifacts() {
+  tmp_settings="tmp-settings.xml"
+  echo "<settings><servers><server>" > $tmp_settings
+  echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
+  echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
+  echo "</server></servers></settings>" >> $tmp_settings
+
+  ./build/mvn --settings $tmp_settings -DskipTests "${PALANTIR_FLAGS[@]}" deploy
+}
+
+make_dist() {
+  build_flags="$1"
+  shift 1
+  version=$(get_version)
+  hadoop_name="hadoop-palantir"
+  artifact_name="spark-dist_2.11-${hadoop_name}"
+  file_name="spark-dist-${version}-${hadoop_name}.tgz"
+  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
+  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "$file_name" "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz"
+  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST "https://api.bintray.com/content/palantir/releases/spark/${version}/publish"
+}
diff --git a/dev/set_version_and_package.sh b/dev/set_version_and_package.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
+
+source "$FWDIR/publish_functions.sh"
+
+set_version_and_package