From 3bbbf9debc869c4abfb0eae909cac465bf242ade Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Fri, 7 Aug 2020 09:41:09 -0400 Subject: [PATCH] Update CI for python 3 --- .travis.yml | 5 +++ scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake | 2 -- .../ci-el7-gnu8-openmpi-ohpc-static.cmake | 4 +-- .../ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake | 5 ++- scripts/ci/cmake/ci-el7-intel-ohpc.cmake | 4 +-- .../ci/cmake/ci-el7-intel-openmpi-ohpc.cmake | 1 - scripts/ci/cmake/ci-el7-intel18-ohpc.cmake | 4 +-- .../cmake/ci-el7-intel18-openmpi-ohpc.cmake | 1 - scripts/ci/images/build-emu-power8-images.sh | 16 +++++----- scripts/ci/images/build-native-images.sh | 16 ++++++++-- scripts/ci/images/el7-base/Dockerfile | 7 ++-- .../ci/images/el7-gnu8-ohpc-base/Dockerfile | 15 +++++---- scripts/ci/images/el7-gnu8-ohpc/Dockerfile | 2 +- .../images/el7-gnu8-openmpi-ohpc/Dockerfile | 6 ++-- .../ci/images/el7-intel-ohpc-base/Dockerfile | 21 ++++++------ scripts/ci/images/el7-intel-ohpc/Dockerfile | 2 +- .../images/el7-intel-openmpi-ohpc/Dockerfile | 6 ++-- scripts/ci/images/el7/Dockerfile | 14 ++++---- scripts/ci/images/emu-el7-base/Dockerfile | 6 ++-- .../ci/images/power8-el7-pgi-base/Dockerfile | 14 ++++---- .../ci/images/power8-el7-xl-base/Dockerfile | 14 ++++---- scripts/ci/images/suse-pgi-base/.gitignore | 1 + scripts/ci/images/suse-pgi-base/Dockerfile | 32 +++++++++---------- scripts/travis/run-format.sh | 2 +- 24 files changed, 108 insertions(+), 92 deletions(-) create mode 100644 scripts/ci/images/suse-pgi-base/.gitignore diff --git a/.travis.yml b/.travis.yml index f5876f2922..72d504dd3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,11 @@ sudo: false language: cpp dist: xenial +addons: + apt: + update: true + packages: + - python3-pip env: matrix: - BUILD_MATRIX_ENTRY=format diff --git a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake index 6b36737ba4..6bbc905f82 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake index 6097378c81..218fa54056 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake @@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load gnu8) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load openmpi3) env_module(load phdf5) -env_module(load py2-mpi4py) +env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake index e78dd52fe8..16f5ccf725 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake @@ -8,10 +8,10 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load gnu8) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load openmpi3) env_module(load phdf5) -env_module(load py2-mpi4py) +env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:BOOL=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake index 6e2af45bad..6aec07e815 100644 --- a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake @@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load intel) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load hdf5) set(ENV{CC} icc) @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake index 0eeef5c7d7..696ca55aad 100644 --- a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake index 6e2af45bad..6aec07e815 100644 --- a/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel18-ohpc.cmake @@ -4,7 +4,7 @@ find_package(EnvModules REQUIRED) env_module(purge) env_module(load intel) -env_module(load py2-numpy) +env_module(load py3-numpy) env_module(load hdf5) set(ENV{CC} icc) @@ -25,8 +25,6 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON - -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake index 0eeef5c7d7..696ca55aad 100644 --- a/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel18-openmpi-ohpc.cmake @@ -34,7 +34,6 @@ ADIOS2_USE_ZFP:STRING=ON MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} -PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3.4 ") set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) diff --git a/scripts/ci/images/build-emu-power8-images.sh b/scripts/ci/images/build-emu-power8-images.sh index ae60bda032..f138e55ee1 100755 --- a/scripts/ci/images/build-emu-power8-images.sh +++ b/scripts/ci/images/build-emu-power8-images.sh @@ -3,7 +3,7 @@ ######################################## # ppc64le CentOS 7 emulation base image ######################################## -docker build --rm --squash \ +docker build --squash \ --build-arg TARGET_ARCH_SYSTEM=ppc64le \ --build-arg TARGET_ARCH_DOCKER=ppc64le \ --build-arg TARGET_CPU=power8 \ @@ -17,7 +17,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7 ######################################## # ppc64le CI base image ######################################## -docker build --rm --squash \ +docker build --squash \ --build-arg TARGET_CPU=power8 \ -t ornladios/adios2:ci-x86_64-power8-el7-base \ emu-el7-base @@ -33,7 +33,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-base ######################################## # XL base image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl-base \ power8-el7-xl-base docker-squash \ @@ -48,7 +48,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-base ######################################## # XL builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl \ --build-arg COMPILER=xl \ power8-el7-leaf @@ -64,7 +64,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl ######################################## # XL + MPI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-xl-smpi \ --build-arg COMPILER=xl \ power8-el7-leaf-smpi @@ -80,7 +80,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-xl-smpi ######################################## # PGI base image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ power8-el7-pgi-base docker-squash \ @@ -95,7 +95,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi-base ######################################## # PGI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi \ --build-arg COMPILER=pgi \ power8-el7-leaf @@ -111,7 +111,7 @@ docker push ornladios/adios2:ci-x86_64-power8-el7-pgi ######################################## # PGI + MPI builder image ######################################## -docker build --rm \ +docker build \ -t ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi \ --build-arg COMPILER=pgi --build-arg HDF5_ARGS="-DMPI_C_COMPILER=mpipgicc" \ power8-el7-leaf-smpi diff --git a/scripts/ci/images/build-native-images.sh b/scripts/ci/images/build-native-images.sh index fe0ab10a36..f80dd5db6d 100755 --- a/scripts/ci/images/build-native-images.sh +++ b/scripts/ci/images/build-native-images.sh @@ -14,7 +14,8 @@ function build_partially_squashed_image() local IMAGE_FROM=$1 local IMAGE_TO=$2 - docker build --rm -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO} + echo "${IMAGE_TO}" + docker build -t ornladios/adios2:ci-${IMAGE_TO} ${IMAGE_TO} docker-squash \ -f ornladios/adios2:ci-${IMAGE_FROM} \ @@ -22,6 +23,9 @@ function build_partially_squashed_image() ornladios/adios2:ci-${IMAGE_TO} } +if [ "${ADIOS2_DOCKER_BUILD}" != "0" ] +then + echo "************************************************************" echo "* Building fully squashed root base images *" echo "************************************************************" @@ -29,7 +33,7 @@ ROOT_BASE_IMAGES="el7-base suse-pgi-base fedora-sanitizers-base debian-sid" for IMAGE in ${ROOT_BASE_IMAGES} do echo "${IMAGE}" - docker build --squash --rm -t ornladios/adios2:ci-${IMAGE} ${IMAGE} + docker build --squash -t ornladios/adios2:ci-${IMAGE} ${IMAGE} echo done @@ -55,6 +59,12 @@ do echo done +fi + + +if [ "${ADIOS2_DOCKER_PUSH}" != "0" ] +then + echo "************************************************************" echo "* Push all images *" echo "************************************************************" @@ -65,3 +75,5 @@ do docker push ornladios/adios2:ci-${IMAGE} echo done + +fi diff --git a/scripts/ci/images/el7-base/Dockerfile b/scripts/ci/images/el7-base/Dockerfile index 649ed6efde..fea462b765 100644 --- a/scripts/ci/images/el7-base/Dockerfile +++ b/scripts/ci/images/el7-base/Dockerfile @@ -5,13 +5,14 @@ RUN yum upgrade -y && \ yum -y install make curl file valgrind vim bison flex sudo gdb \ pkgconfig bison flex pkgconfig gcc gcc-c++ gcc-gfortran \ zlib zlib-devel bzip2 bzip2-libs bzip2-devel libpng-devel \ - libfabric-devel libffi-devel + libfabric-devel libffi-devel python3 python3-devel RUN yum -y install epel-release && \ yum -y install zeromq-devel blosc-devel libzstd-devel # Install and setup newer version of git -RUN yum install -y https://centos7.iuscommunity.org/ius-release.rpm && \ - yum -y install git222 +RUN yum install -y https://repo.ius.io/ius-release-el7.rpm && \ + yum -y install git224 && \ + yum remove -y ius-release # Install the most recent CMake nightly binary WORKDIR /opt/cmake diff --git a/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile index ef1ae65299..4da24cdf62 100644 --- a/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile +++ b/scripts/ci/images/el7-gnu8-ohpc-base/Dockerfile @@ -2,7 +2,8 @@ FROM ornladios/adios2:ci-el7-base # Install OpenHPC packages RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ - yum -y install lmod-ohpc gnu8-compilers-ohpc + yum -y install lmod-ohpc gnu8-compilers-ohpc python3-numpy-gnu8-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/* # Install ZFP WORKDIR /opt/zfp @@ -22,19 +23,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ source /etc/profile && \ module load gnu8 && \ export CC=gcc CXX=g++ FC=gfortran && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rfv \ diff --git a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile index 360b646e82..905e26df46 100644 --- a/scripts/ci/images/el7-gnu8-ohpc/Dockerfile +++ b/scripts/ci/images/el7-gnu8-ohpc/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-gnu8-ohpc-base # Install OpenHPC packages -RUN yum -y install python34-devel python34-numpy-gnu8-ohpc hdf5-gnu8-ohpc +RUN yum -y install hdf5-gnu8-ohpc # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile index 76ff59334a..3b81982bdb 100644 --- a/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile +++ b/scripts/ci/images/el7-gnu8-openmpi-ohpc/Dockerfile @@ -1,8 +1,10 @@ FROM ornladios/adios2:ci-el7-gnu8-ohpc-base # Install OpenHPC packages -RUN yum -y install python-devel python-numpy-gnu8-ohpc openmpi3-gnu8-ohpc \ - phdf5-gnu8-openmpi3-ohpc python-mpi4py-gnu8-openmpi3-ohpc +RUN yum -y install openmpi3-gnu8-ohpc phdf5-gnu8-openmpi3-ohpc \ + python3-mpi4py-gnu8-openmpi3-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/* + # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-intel-ohpc-base/Dockerfile b/scripts/ci/images/el7-intel-ohpc-base/Dockerfile index 560417479e..fcd817a38b 100644 --- a/scripts/ci/images/el7-intel-ohpc-base/Dockerfile +++ b/scripts/ci/images/el7-intel-ohpc-base/Dockerfile @@ -33,7 +33,7 @@ FROM ornladios/adios2:ci-el7-base # Install Intel C++ and Fortran compilers ARG LICENSE_FILE=CI.lic -ARG INTEL_VERSION=2020_update1 +ARG INTEL_VERSION=2020_update2 WORKDIR /tmp COPY ${LICENSE_FILE} /opt/intel/licenses/license.lic COPY silent-cpp.cfg /tmp @@ -46,8 +46,11 @@ RUN tar -xzf parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_cpp.tgz && ./parallel_studio_xe_${INTEL_VERSION}_composer_edition_for_fortran/install.sh -s ./silent-fortran.cfg # Install OpenHPC packages -RUN yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ - yum -y install lmod-ohpc intel-compilers-devel-ohpc +RUN yum install -y /tmp/*_for_cpp/rpm/intel-{comp,ps}xe-doc-*.rpm && \ + yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm && \ + yum -y install lmod-ohpc intel-compilers-devel-ohpc \ + python3-numpy-intel-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-numpy/* # Install ZFP WORKDIR /opt/zfp @@ -67,19 +70,19 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ source /etc/profile && \ module load intel && \ export CC=icc CXX=icpc FC=ifort && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rfv \ diff --git a/scripts/ci/images/el7-intel-ohpc/Dockerfile b/scripts/ci/images/el7-intel-ohpc/Dockerfile index 09ed2189f1..0b12609af9 100644 --- a/scripts/ci/images/el7-intel-ohpc/Dockerfile +++ b/scripts/ci/images/el7-intel-ohpc/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-intel-ohpc-base # Install OpenHPC packages -RUN yum -y install python-devel python-numpy-intel-ohpc hdf5-intel-ohpc +RUN yum -y install hdf5-intel-ohpc # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile b/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile index 1f0b46808e..752fd0224c 100644 --- a/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile +++ b/scripts/ci/images/el7-intel-openmpi-ohpc/Dockerfile @@ -1,9 +1,9 @@ FROM ornladios/adios2:ci-el7-intel-ohpc-base # Install OpenHPC packages -RUN yum -y install python34-devel python34-numpy-intel-ohpc \ - openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \ - python34-mpi4py-intel-openmpi3-ohpc +RUN yum -y install openmpi3-intel-ohpc phdf5-intel-openmpi3-ohpc \ + python3-mpi4py-intel-openmpi3-ohpc && \ + sed 's|python3\.4|python3.6|g' -i /opt/ohpc/pub/moduledeps/*/py3-mpi4py/* # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/el7/Dockerfile b/scripts/ci/images/el7/Dockerfile index f9f0e55de2..b6476d3089 100644 --- a/scripts/ci/images/el7/Dockerfile +++ b/scripts/ci/images/el7/Dockerfile @@ -1,7 +1,7 @@ FROM ornladios/adios2:ci-el7-base # Install extra system dev packages -RUN yum -y install hdf5-devel python-devel numpy +RUN yum -y install hdf5-devel python36-numpy # Install ZFP WORKDIR /opt/zfp @@ -18,16 +18,16 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 ../SZ-2.1.8.0 && \ + cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH /opt/sz/2.1.8.0/bin:${PATH} -ENV LD_LIBRARY_PATH /opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} -ENV CMAKE_PREFIX_PATH /opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH /opt/sz/2.1.8.3/bin:${PATH} +ENV LD_LIBRARY_PATH /opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} +ENV CMAKE_PREFIX_PATH /opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/emu-el7-base/Dockerfile b/scripts/ci/images/emu-el7-base/Dockerfile index 1dc5ea759f..370815c529 100644 --- a/scripts/ci/images/emu-el7-base/Dockerfile +++ b/scripts/ci/images/emu-el7-base/Dockerfile @@ -30,7 +30,7 @@ RUN yum install -y \ xz-devel rhash-devel zlib-devel libzstd-devel && \ mkdir tmp && \ cd tmp && \ - curl -L https://github.com/Kitware/CMake/releases/download/v3.17.0/cmake-3.17.0.tar.gz | \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz | \ tar -xz && \ mkdir build && \ cd build && \ @@ -40,12 +40,12 @@ RUN yum install -y \ --no-system-libarchive \ --no-system-libuv \ --no-system-jsoncpp \ - --prefix=/opt/cmake/3.17.0 \ + --prefix=/opt/cmake/3.18.0 \ --parallel=$(grep -c '^processor' /proc/cpuinfo) && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd ../.. && \ rm -rf tmp -ENV PATH=/opt/cmake/3.17.0/bin:${PATH} +ENV PATH=/opt/cmake/3.18.0/bin:${PATH} # Misc cleanup of unneeded files RUN yum clean all && \ diff --git a/scripts/ci/images/power8-el7-pgi-base/Dockerfile b/scripts/ci/images/power8-el7-pgi-base/Dockerfile index 1c803c90fd..554b7bf0ba 100644 --- a/scripts/ci/images/power8-el7-pgi-base/Dockerfile +++ b/scripts/ci/images/power8-el7-pgi-base/Dockerfile @@ -48,20 +48,20 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ + ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* diff --git a/scripts/ci/images/power8-el7-xl-base/Dockerfile b/scripts/ci/images/power8-el7-xl-base/Dockerfile index 7395debf22..fe70270954 100644 --- a/scripts/ci/images/power8-el7-xl-base/Dockerfile +++ b/scripts/ci/images/power8-el7-xl-base/Dockerfile @@ -39,20 +39,20 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ + ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* diff --git a/scripts/ci/images/suse-pgi-base/.gitignore b/scripts/ci/images/suse-pgi-base/.gitignore new file mode 100644 index 0000000000..9e4c8aaaf6 --- /dev/null +++ b/scripts/ci/images/suse-pgi-base/.gitignore @@ -0,0 +1 @@ +pgilinux*.tar.gz diff --git a/scripts/ci/images/suse-pgi-base/Dockerfile b/scripts/ci/images/suse-pgi-base/Dockerfile index 3fae6328e4..a9861bfef0 100644 --- a/scripts/ci/images/suse-pgi-base/Dockerfile +++ b/scripts/ci/images/suse-pgi-base/Dockerfile @@ -4,8 +4,8 @@ FROM opensuse/leap:15.2 RUN zypper addrepo -fc https://download.opensuse.org/repositories/devel:/tools:/scm/openSUSE_Leap_15.2/devel:tools:scm.repo RUN zypper --gpg-auto-import-keys ref RUN zypper in -y gcc gcc-c++ gcc-fortran git make curl tar f2c glibc-locale \ - glibc-devel libbz2-devel pkg-config libzmq-devel zlib-devel gdb vim valgrind \ - bzip2 gzip blosc-devel libzstd-devel openssl-devel environment-modules + glibc-devel libbz2-devel pkg-config zeromq-devel zlib-devel gdb vim valgrind \ + bzip2 gzip blosc-devel libzstd-devel libopenssl-devel Modules # Workaround so pgi can find g77 WORKDIR /usr/bin @@ -13,19 +13,19 @@ RUN ln -s gfortran g77 # Install PGI compiler WORKDIR /tmp/pgi-install -RUN curl -L https://data.kitware.com/api/v1/file/5defe4f7af2e2eed35763524/download | tar -xz -RUN export \ +COPY pgilinux-2019-1910-x86-64.tar.gz /tmp/pgi-install +RUN tar -xzf pgilinux-2019-1910-x86-64.tar.gz && \ + export \ PGI_SILENT=true \ PGI_ACCEPT_EULA=accept \ PGI_INSTALL_DIR=/opt/pgi \ PGI_INSTALL_NVIDIA=false \ PGI_INSTALL_JAVA=false \ PGI_INSTALL_MPI=false \ - PGI_MPI_GPU_SUPPORT=false \ - && ./install - -RUN echo 'export MODULEPATH=/opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.sh \ - && echo 'setenv MODULEPATH /opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.csh + PGI_MPI_GPU_SUPPORT=false && \ + ./install && \ + echo 'export MODULEPATH=/opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.sh && \ + echo 'setenv MODULEPATH /opt/pgi/modulefiles:${MODULEPATH}' > /etc/profile.d/pgi-modules.csh # Install the most recent CMake nightly binary WORKDIR /opt/cmake @@ -54,7 +54,7 @@ ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ # Install SZ WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz && \ +RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ source /etc/profile && \ @@ -63,14 +63,14 @@ RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.0.tar.gz | tar -xvz /opt/cmake/bin/cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.0 \ - ../SZ-2.1.8.0 && \ + -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ + ../SZ-2.1.8.3 && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd .. && \ - rm -rf SZ-2.1.8.0 build -ENV PATH=/opt/sz/2.1.8.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.0/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.0:${CMAKE_PREFIX_PATH} + rm -rf SZ-2.1.8.3 build +ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} # Misc cleanup of unneeded files RUN rm -rf /tmp/* && \ diff --git a/scripts/travis/run-format.sh b/scripts/travis/run-format.sh index 8395840ae9..619664e482 100755 --- a/scripts/travis/run-format.sh +++ b/scripts/travis/run-format.sh @@ -20,7 +20,7 @@ git config --global clangformat.binary ${HOME}/clang+llvm-7.0.1-x86_64-linux-gnu popd # Install flake8 -pip install --user flake8 +pip3 install --user flake8 cd ${SOURCE_DIR}