Test PySpark using PyArrow 1.x (palantir#48)

16pierre · Apr 22, 2021 · 6f3a5ef · 6f3a5ef
1 parent 12a4d01
commit 6f3a5ef
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 14 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,11 +2,12 @@ version: 2
 
 defaults: &defaults
   docker:
-    - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-base:0.2.4
+    - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-base:0.2.5
   resource_class: xlarge
   environment: &defaults-environment
     TERM: dumb
     BUILD_SBT_CACHE: "/home/circleci/build-sbt-cache"
+    APACHE_MIRROR: https://artifactory.palantir.build/artifactory/external-dist-apache
 
 test-defaults: &test-defaults
   <<: *defaults
@@ -124,7 +125,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       # Saves us from recompiling every time...
       # (And causes compilation failures.)
@@ -257,7 +258,7 @@ jobs:
     # depends on build-sbt, but we only need the assembly jars
     <<: *defaults
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-python:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-python:0.2.5
     parallelism: 2
     steps:
       - *checkout-code
@@ -281,7 +282,7 @@ jobs:
     # depends on build-sbt, but we only need the assembly jars
     <<: *defaults
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       - *checkout-code
       - attach_workspace:
@@ -392,7 +393,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       - *checkout-code
       - run:
@@ -413,7 +414,7 @@ jobs:
   deploy-spark-docker-resources:
     <<: *defaults
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       - *checkout-code
       - deploy:
@@ -423,7 +424,7 @@ jobs:
     <<: *defaults
     # Some part of the maven setup fails if there's no R, so we need to use the R image here
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       # This cache contains the whole project after version was set and mvn package was called
       # Restoring first (and instead of checkout) as mvn versions:set mutates real source code...
@@ -442,7 +443,7 @@ jobs:
     # This task is dedicated to the dist publishing.
     <<: *defaults
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     steps:
       - restore_cache:
           key: v1-maven-build-with-version-{{ .Branch }}-{{ .Revision }}
@@ -457,10 +458,10 @@ jobs:
 
   circle-all:
     docker:
-      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.4
+      - image: hub.docker.palantir.build/palantirtechnologies/circle-spark-r:0.2.5
     resource_class: small
     steps:
-      - run: {command: echo "All required jobs finished successfully"}
+      - run: { command: echo "All required jobs finished successfully" }
 
 workflows:
   version: 2

diff --git a/dev/docker-images/Makefile b/dev/docker-images/Makefile
@@ -17,7 +17,7 @@
 
 .PHONY: all publish base python r
 
-VERSION=0.2.4
+VERSION=0.2.5
 BASE_IMAGE_NAME = "palantirtechnologies/circle-spark-base:${VERSION}"
 PYTHON_IMAGE_NAME = "palantirtechnologies/circle-spark-python:${VERSION}"
 R_IMAGE_NAME = "palantirtechnologies/circle-spark-r:${VERSION}"

diff --git a/dev/docker-images/python/Dockerfile b/dev/docker-images/python/Dockerfile
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM palantirtechnologies/circle-spark-base:0.2.4
+FROM palantirtechnologies/circle-spark-base:0.2.5
 
 # Install pyenv
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
@@ -24,7 +24,7 @@ ENV PATH="$CIRCLE_HOME/.pyenv/shims:$CIRCLE_HOME/.pyenv/bin:$PATH"
 RUN mkdir -p $(pyenv root)/versions \
   && ln -s $CONDA_ROOT $(pyenv root)/versions/our-miniconda \
   && $CONDA_BIN create -y -n python2 -c anaconda -c conda-forge python==2.7.18 pyarrow==0.15.1 pandas==0.23.2 nomkl \
-  && $CONDA_BIN create -y -n python3 -c anaconda -c conda-forge python=3.6 pyarrow==0.15.1 pandas==0.23.2 nomkl \
+  && $CONDA_BIN create -y -n python3 -c anaconda -c conda-forge python=3.6 pyarrow==1.0.1 pandas==0.23.2 nomkl \
   && $CONDA_BIN clean --all
 
 RUN pyenv global our-miniconda/envs/python2 our-miniconda/envs/python3 \

diff --git a/dev/docker-images/r/Dockerfile b/dev/docker-images/r/Dockerfile
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM palantirtechnologies/circle-spark-base:0.2.4
+FROM palantirtechnologies/circle-spark-base:0.2.5
 
 # Install R and SparkR dependencies
 # Rscript -e "install.packages(c('knitr', 'rmarkdown', 'devtools', 'testthat', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"