Update CI for python 3

ornladios · Aug 7, 2020 · 3bbbf9d · 3bbbf9d
1 parent ce0eb75
commit 3bbbf9d
Show file tree

Hide file tree

Showing 24 changed files with 108 additions and 92 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,11 @@
 sudo: false
 language: cpp
 dist: xenial
+addons:
+  apt:
+    update: true
+    packages:
+      - python3-pip
 env:
   matrix:
     - BUILD_MATRIX_ENTRY=format

diff --git a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake
@@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON
 ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
 ADIOS2_USE_ZFP:BOOL=ON
-
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4
 ")
 
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")

diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake
@@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED)
 
 env_module(purge)
 env_module(load gnu8)
-env_module(load py2-numpy)
+env_module(load py3-numpy)
 env_module(load openmpi3)
 env_module(load phdf5)
-env_module(load py2-mpi4py)
+env_module(load py3-mpi4py)
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)

diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake
@@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED)
 
 env_module(purge)
 env_module(load gnu8)
-env_module(load py2-numpy)
+env_module(load py3-numpy)
 env_module(load openmpi3)
 env_module(load phdf5)
-env_module(load py2-mpi4py)
+env_module(load py3-mpi4py)
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)
@@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:BOOL=ON
 
 MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7
 ")
 
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 1)

diff --git a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake
@@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED)
 
 env_module(purge)
 env_module(load intel)
-env_module(load py2-numpy)
+env_module(load py3-numpy)
 env_module(load hdf5)
 
 set(ENV{CC}  icc)
@@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON
 ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
 ADIOS2_USE_ZFP:STRING=ON
-
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7
 ")
 
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")

diff --git a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake
@@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON
 
 MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4
 ")
 
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 1)

diff --git a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake
@@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED)
 
 env_module(purge)
 env_module(load intel)
-env_module(load py2-numpy)
+env_module(load py3-numpy)
 env_module(load hdf5)
 
 set(ENV{CC}  icc)
@@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON
 ADIOS2_USE_SZ:BOOL=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
 ADIOS2_USE_ZFP:STRING=ON
-
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7
 ")
 
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")

diff --git a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake
@@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON
 
 MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe
 MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS}
-PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4
 ")
 
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 1)

diff --git a/scripts/ci/images/build-emu-power8-images.sh b/scripts/ci/images/build-emu-power8-images.sh
@@ -3,7 +3,7 @@
 ########################################
 # ppc64le CentOS 7 emulation base image
 ########################################
-docker build --rm --squash \
+docker build --squash \
   --build-arg TARGET_ARCH_SYSTEM=ppc64le \
   --build-arg TARGET_ARCH_DOCKER=ppc64le \
   --build-arg TARGET_CPU=power8 \
@@ -17,7 +17,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7
 ########################################
 # ppc64le CI base image
 ########################################
-docker build --rm --squash \
+docker build --squash \
   --build-arg TARGET_CPU=power8 \
   -t ornladios/adios2:ci-x86_64-power8-el7-base \
   emu-el7-base
@@ -33,7 +33,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-base
 ########################################
 # XL base image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-xl-base \
   power8-el7-xl-base
 docker-squash \
@@ -48,7 +48,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-base
 ########################################
 # XL builder image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-xl \
   --build-arg COMPILER=xl \
   power8-el7-leaf
@@ -64,7 +64,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl
 ########################################
 # XL + MPI builder image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-xl-smpi \
   --build-arg COMPILER=xl \
   power8-el7-leaf-smpi
@@ -80,7 +80,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-smpi
 ########################################
 # PGI base image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-pgi-base \
   power8-el7-pgi-base
 docker-squash \
@@ -95,7 +95,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi-base
 ########################################
 # PGI builder image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-pgi \
   --build-arg COMPILER=pgi \
   power8-el7-leaf
@@ -111,7 +111,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi
 ########################################
 # PGI + MPI builder image
 ########################################
-docker build --rm \
+docker build \
   -t ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi \
   --build-arg COMPILER=pgi --build-arg HDF5_ARGS="-DMPI_C_COMPILER=mpipgicc" \
   power8-el7-leaf-smpi

diff --git a/scripts/ci/images/build-native-images.sh b/scripts/ci/images/build-native-images.sh
@@ -14,22 +14,26 @@ function build_partially_squashed_image()
   local IMAGE_FROM=$1
   local IMAGE_TO=$2
 
-  docker build --rm -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO}
+  echo "${IMAGE_TO}"
+  docker build -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO}
 
   docker-squash \
     -f ornladios/adios2:ci-${IMAGE_FROM} \
     -t ornladios/adios2:ci-${IMAGE_TO} \
     ornladios/adios2:ci-${IMAGE_TO}
 }
 
+if [ "${ADIOS2_DOCKER_BUILD}" != "0" ]
+then
+
 echo "************************************************************"
 echo "* Building fully squashed root base images                 *"
 echo "************************************************************"
 ROOT_BASE_IMAGES="el7-base suse-pgi-base fedora-sanitizers-base debian-sid"
 for IMAGE in ${ROOT_BASE_IMAGES}
 do
   echo "${IMAGE}"
-  docker build --squash --rm -t ornladios/adios2:ci-${IMAGE} ${IMAGE}
+  docker build --squash -t ornladios/adios2:ci-${IMAGE} ${IMAGE}
   echo
 done
 
@@ -55,6 +59,12 @@ do
   echo
 done
 
+fi
+
+
+if [ "${ADIOS2_DOCKER_PUSH}" != "0" ]
+then
+
 echo "************************************************************"
 echo "* Push all images                                          *"
 echo "************************************************************"
@@ -65,3 +75,5 @@ do
   docker push ornladios/adios2:ci-${IMAGE}
   echo
 done
+
+fi
diff --git a/scripts/ci/images/el7-base/Dockerfile b/scripts/ci/images/el7-base/Dockerfile
@@ -5,13 +5,14 @@ RUN yum upgrade -y && \
     yum -y install make curl file valgrind vim bison flex sudo gdb \
         pkgconfig bison flex pkgconfig gcc gcc-c++ gcc-gfortran \
         zlib zlib-devel bzip2 bzip2-libs bzip2-devel libpng-devel \
-        libfabric-devel libffi-devel
+        libfabric-devel libffi-devel python3 python3-devel
 RUN yum -y install epel-release && \
     yum -y install zeromq-devel blosc-devel libzstd-devel
 
 # Install and setup newer version of git
-RUN yum install -y https://centos7.iuscommunity.org/ius-release.rpm && \
-    yum -y install git222
+RUN yum install -y https://repo.ius.io/ius-release-el7.rpm && \
+    yum -y install git224 && \
+    yum remove -y ius-release
 
 # Install the most recent CMake nightly binary
 WORKDIR /opt/cmake

diff --git a/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile
@@ -2,7 +2,8 @@ FROM ornladios/adios2:ci-el7-base
 
 # Install OpenHPC packages
 RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \
-    yum -y install lmod-ohpc gnu8-compilers-ohpc
+    yum -y install lmod-ohpc gnu8-compilers-ohpc python3-numpy-gnu8-ohpc && \
+    sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/*
 
 # Install ZFP
 WORKDIR /opt/zfp
@@ -22,19 +23,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \
 
 # Install SZ
 WORKDIR /opt/sz
-RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \
+RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \
     mkdir build && \
     cd build && \
     source /etc/profile && \
     module load gnu8 && \
     export CC=gcc CXX=g++ FC=gfortran && \
-    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \
     make -j$(grep -c '^processor' /proc/cpuinfo) install && \
     cd .. && \
-    rm -rf SZ-2.1.8.0 build
-ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \
-    LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \
-    CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH}
+    rm -rf SZ-2.1.8.3 build
+ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \
+    LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \
+    CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH}
 
 # Misc cleanup of unneeded files
 RUN rm -rfv \

diff --git a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile
@@ -1,7 +1,7 @@
 FROM ornladios/adios2:ci-el7-gnu8-ohpc-base
 
 # Install OpenHPC packages
-RUN yum -y install python34-devel python34-numpy-gnu8-ohpc hdf5-gnu8-ohpc
+RUN yum -y install hdf5-gnu8-ohpc
 
 # Misc cleanup of unneeded files
 RUN yum clean all && \

diff --git a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile
@@ -1,8 +1,10 @@
 FROM ornladios/adios2:ci-el7-gnu8-ohpc-base
 
 # Install OpenHPC packages
-RUN yum -y install python-devel python-numpy-gnu8-ohpc openmpi3-gnu8-ohpc \
-        phdf5-gnu8-openmpi3-ohpc python-mpi4py-gnu8-openmpi3-ohpc
+RUN yum -y install openmpi3-gnu8-ohpc phdf5-gnu8-openmpi3-ohpc \
+        python3-mpi4py-gnu8-openmpi3-ohpc && \
+    sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/*
+
 
 # Misc cleanup of unneeded files
 RUN yum clean all && \

diff --git a/scripts/ci/images/el7-intel-ohpc-base/Dockerfile b/scripts/ci/images/el7-intel-ohpc-base/Dockerfile
@@ -33,7 +33,7 @@ FROM ornladios/adios2:ci-el7-base
 
 # Install Intel C++ and Fortran compilers
 ARG LICENSE_FILE=CI.lic
-ARG INTEL_VERSION=2020_update1
+ARG INTEL_VERSION=2020_update2
 WORKDIR /tmp
 COPY ${LICENSE_FILE} /opt/intel/licenses/license.lic
 COPY silent-cpp.cfg /tmp
@@ -46,8 +46,11 @@ RUN tar -xzf parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_cpp.tgz &&
     ./parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_fortran/install.sh -s ./silent-fortran.cfg
 
 # Install OpenHPC packages
-RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \
-    yum -y install lmod-ohpc intel-compilers-devel-ohpc
+RUN yum install -y /tmp/*_for_cpp/rpm/intel-{comp,ps}xe-doc-*.rpm && \
+    yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \
+    yum -y install lmod-ohpc intel-compilers-devel-ohpc \
+        python3-numpy-intel-ohpc && \
+    sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/*
 
 # Install ZFP
 WORKDIR /opt/zfp
@@ -67,19 +70,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \
 
 # Install SZ
 WORKDIR /opt/sz
-RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \
+RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \
     mkdir build && \
     cd build && \
     source /etc/profile && \
     module load intel && \
     export CC=icc CXX=icpc FC=ifort && \
-    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \
+    cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \
     make -j$(grep -c '^processor' /proc/cpuinfo) install && \
     cd .. && \
-    rm -rf SZ-2.1.8.0 build
-ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \
-    LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \
-    CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH}
+    rm -rf SZ-2.1.8.3 build
+ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \
+    LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \
+    CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH}
 
 # Misc cleanup of unneeded files
 RUN rm -rfv \

diff --git a/scripts/ci/images/el7-intel-ohpc/Dockerfile b/scripts/ci/images/el7-intel-ohpc/Dockerfile
@@ -1,7 +1,7 @@
 FROM ornladios/adios2:ci-el7-intel-ohpc-base
 
 # Install OpenHPC packages
-RUN yum -y install python-devel python-numpy-intel-ohpc hdf5-intel-ohpc
+RUN yum -y install hdf5-intel-ohpc
 
 # Misc cleanup of unneeded files
 RUN yum clean all && \

diff --git a/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile
@@ -1,9 +1,9 @@
 FROM ornladios/adios2:ci-el7-intel-ohpc-base
 
 # Install OpenHPC packages
-RUN yum -y install python34-devel python34-numpy-intel-ohpc \
-        openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \
-        python34-mpi4py-intel-openmpi3-ohpc
+RUN yum -y install openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \
+        python3-mpi4py-intel-openmpi3-ohpc && \
+    sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/*
 
 # Misc cleanup of unneeded files
 RUN yum clean all && \