Merge pull request #3537 from vicentebolea/enable-linux-emu-builds

Enable Linux Emulation build (Power8, IBM XL)

.github/workflows/everything.yml

@@ -145,7 +145,11 @@ jobs:
 
   linux_emu:
     needs: [format, git_checks]
-    if: false # needs.git_checks.outputs.num_code_changes > 0
+    if: needs.git_checks.outputs.num_code_changes > 0 &&
+        (
+        contains('refs/heads/master', github.ref) ||
+        contains('refs/heads/release_', github.ref)
+        )
 
     runs-on: ubuntu-latest
     container:
@@ -164,20 +168,14 @@ jobs:
         cpu: [power8]
         os: [el7]
         compiler: [xl]
-        parallel: [serial, mpi]
+        parallel: [serial]
         include:
           - cpu: power8
             os: el7
             compiler: xl
             parallel: serial
             container: ornladios/adios2:ci-x86_64-power8-el7-xl
             arch: ppc64le
-          - cpu: power8
-            os: el7
-            compiler: xl
-            parallel: mpi
-            container: ornladios/adios2:ci-x86_64-power8-el7-xl-smpi
-            arch: ppc64le
 
     steps:
       - name: Emulation Setup

scripts/ci/cmake-v2/ci-power8-el7-xl-serial.cmake

@@ -1,4 +1,4 @@
-# Client maintainer: chuck.atkins@kitware.com
+# Client maintainer: vicente.bolea@kitware.com
 
 set(ENV{CC}  xlc)
 set(ENV{CXX} xlc++)
@@ -20,6 +20,7 @@ ADIOS2_USE_ZFP:BOOL=ON
 set(CTEST_TEST_ARGS
   INCLUDE "^(Interface|Install)"
 )
+
 set(CTEST_TEST_TIMEOUT 300)
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")

scripts/ci/gh-actions/linux-setup.sh

@@ -3,7 +3,7 @@
 set -ex
 
 export CI_ROOT_DIR="${GITHUB_WORKSPACE}/.."
-export CI_SOURCE_DIR="${GITHUB_WORKSPACE}"
+export CI_SOURCE_DIR="${GITHUB_WORKSPACE}/source"
 
 declare -r local_scripts_dir="$(dirname -- $0)/config"
 

scripts/ci/images/power8-el7-xl-base/.gitignore → scripts/ci/images/.gitignore

@@ -1,2 +1,3 @@
+*-stamp
 *.tar.gz
 *.rpm

scripts/ci/images/build-emu-power8-image.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -ex
+
+########################################
+# ppc64le CentOS 7 emulation base image
+########################################
+docker build --squash \
+  --build-arg TARGET_ARCH_SYSTEM=ppc64le \
+  --build-arg TARGET_ARCH_DOCKER=ppc64le \
+  --build-arg TARGET_CPU=power8 \
+  -t ornladios/adios2:ci-x86_64-power8-el7 \
+  emu-el7
+echo ""
+echo "Pushing ornladios/adios2:ci-x86_64-power8-el7"
+echo ""
+docker push ornladios/adios2:ci-x86_64-power8-el7
+
+######################################
+ ppc64le CI base image
+######################################
+docker build \
+  --build-arg TARGET_CPU=power8 \
+  -t ornladios/adios2:ci-x86_64-power8-el7-base \
+  emu-el7-base
+docker-squash \
+  -f ornladios/adios2:ci-x86_64-power8-el7 \
+  -t ornladios/adios2:ci-x86_64-power8-el7-base \
+  ornladios/adios2:ci-x86_64-power8-el7-base
+echo ""
+echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-base"
+echo ""
+docker push ornladios/adios2:ci-x86_64-power8-el7-base

scripts/ci/images/emu-el7-base/Dockerfile

@@ -3,50 +3,80 @@ FROM ornladios/adios2:ci-x86_64-${TARGET_CPU}-el7
 
 # Install core dev packages
 RUN yum upgrade -y && \
-    yum -y install make curl file valgrind vim bison flex sudo gdb \
-        pkgconfig bison flex pkgconfig gcc gcc-c++ gcc-gfortran \
-        zlib zlib-devel bzip2 bzip2-libs bzip2-devel libpng-devel \
-        libfabric-devel libffi-devel
-RUN yum -y install epel-release && \
-    yum -y install zeromq-devel blosc-devel libzstd-devel
+    yum -y install centos-release-scl && \
+    yum-config-manager --enable rhel-server-rhscl-7-rpms && \
+    yum -y install \
+        bison \
+        bzip2 \
+        bzip2-libs \
+        curl \
+        devtoolset-7 \
+        devtoolset-7-gcc \
+        devtoolset-7-gcc-c++ \
+        devtoolset-7-gcc-gdb \
+        devtoolset-7-gcc-gfortran \
+        file \
+        flex \
+        gcc \
+        gcc-c++ \
+        gcc-gfortran \
+        libfabric-devel \
+        libffi-devel \
+        libpng-devel \
+        make \
+        pkgconfig \
+        sudo \
+        valgrind \
+        vim \
+        zlib \
+    yum -y install epel-release && \
+    yum -y install \
+        blosc-devel \
+        bzip2-devel \
+        curl-devel \
+        expat-devel \
+        gettext \
+        libcurl-devel \
+        libzstd-devel \
+        openssl-devel \
+        rhash-devel \
+        xz-devel \
+        zeromq-devel \
+        zlib-devel && \
+    yum clean all && \
+    rm -rfv /tmp/* /var/cache/yum
+
+ENV LAUNCHER="/usr/bin/scl enable devtoolset-7 --"
 
 # Install and setup newer version of git
 WORKDIR /opt/git
-RUN yum install -y gettext openssl-devel curl-devel expat-devel && \
-    mkdir tmp && \
+RUN mkdir tmp && \
     cd tmp && \
     curl -L https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar -xz && \
     cd git-2.26.0 && \
-    make -j$(grep -c '^processor' /proc/cpuinfo) prefix=/opt/git/2.26.0 all && \
-    make prefix=/opt/git/2.26.0 install && \
+    $LAUNCHER make -j$(grep -c '^processor' /proc/cpuinfo) prefix=/opt/git/2.26.0 all && \
+    $LAUNCHER make prefix=/opt/git/2.26.0 install && \
     cd ../.. && \
     rm -rf tmp
 ENV PATH=/opt/git/2.26.0/bin:${PATH}
 
 # Install the most recent CMake from source
 WORKDIR /opt/cmake
-RUN yum install -y \
-        bzip2-devel libcurl-devel expat-devel \
-        xz-devel rhash-devel zlib-devel libzstd-devel && \
-    mkdir tmp && \
+RUN mkdir tmp && \
     cd tmp && \
-    curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz | \
+    curl -L https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0.tar.gz | \
         tar -xz && \
     mkdir build && \
     cd build && \
-    ../cmake-3.17.0/bootstrap \
+    $LAUNCHER ../cmake-3.26.0/bootstrap \
       --system-libs \
       --no-qt-gui \
       --no-system-libarchive \
       --no-system-libuv \
       --no-system-jsoncpp \
-      --prefix=/opt/cmake/3.18.0 \
+      --prefix=/opt/cmake/3.26.0 \
       --parallel=$(grep -c '^processor' /proc/cpuinfo) && \
     make -j$(grep -c '^processor' /proc/cpuinfo) install && \
     cd ../.. && \
-    rm -rf tmp 
-ENV PATH=/opt/cmake/3.18.0/bin:${PATH}
-
-# Misc cleanup of unneeded files
-RUN yum clean all && \
-    rm -rfv /tmp/* /var/cache/yum
+    rm -rf tmp
+ENV PATH=/opt/cmake/3.26.0/bin:${PATH}

scripts/ci/images/emu-el7/Dockerfile

@@ -35,7 +35,7 @@ FROM ${TARGET_ARCH_DOCKER}/centos:centos7
 # Grab a fully static node.js binary
 # to run various CI actions
 ########################################
-COPY --from=ornladios/adios2:node12-static /node /x86_64/bin/node
+COPY  --from=ornladios/adios2:node12-static /node /x86_64/bin/node
 
 ########################################
 # Build up a minimal busybox shell