CI: major rework for image generations of GHA

CTestCustom.cmake.in

@@ -20,6 +20,7 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION
   "warning generated"
   "warnings generated"
   "warning: template parameter ... is not used in declaring the parameter types of function template"
+  "warning: command-line option '.*' is valid for Fortran but not for C"
 )
 list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
   ".*/thirdparty/.*"

scripts/ci/images-v2/Dockerfile.ci-spack-el8-base

@@ -8,12 +8,11 @@ RUN dnf install -y dnf-plugins-core && \
     dnf install -y epel-release
 RUN dnf install -y \
         bzip2 curl file gcc-c++ gcc git gnupg2 gzip \
-        make patch tcl unzip which xz patchelf diffutils tar Lmod \
+        make patch tcl unzip which xz patchelf diffutils findutils tar Lmod \
         python3 python3-pip
 RUN dnf clean all
 
-RUN git clone -b v0.18.0 https://github.com/spack/spack.git /opt/spack
-COPY packages.yaml.base /etc/spack/packages.yaml
+RUN git clone -b v0.18.1 https://github.com/spack/spack.git /opt/spack
 COPY modules.yaml /etc/spack/modules.yaml
 COPY 0001-spack-rhel-fix.patch /root/0001-spack-rhel-fix.patch
 
@@ -26,5 +25,5 @@ RUN . /opt/spack/share/spack/setup-env.sh && \
     spack compiler find --scope system && \
     spack clean -a && \
     spack mirror add E4S https://cache.e4s.io && \
-    spack buildcache keys -it && \
+    spack buildcache keys --install --trust && \
     spack spec zlib

scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base

@@ -9,5 +9,4 @@ RUN dnf install -y gcc-gfortran && \
 RUN . /opt/spack/share/spack/setup-env.sh && \
     spack compiler rm --scope system gcc && \
     spack compiler add --scope system && \
-    spack config --scope system add "packages:all:compiler:[gcc]" && \
-    spack external find --scope system --not-buildable cuda
+    spack config --scope system add "packages:all:compiler:[gcc]"

scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf

@@ -1,22 +1,27 @@
+# vim: ft=dockerfile
 ARG COMPILER_IMG_BASE
-FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base
+FROM almalinux/almalinux:8 AS builder
 
-ARG MPI_FLAVOR=openmpi
-RUN sed "s|^packages:$|&\n  mpich:\n    version: [3.3.2]\n    variants: device=ch3 netmod=tcp|" \
-      -i /etc/spack/packages.yaml
+WORKDIR /root
+
+RUN dnf update -y && \
+    dnf install -y python3-pip && \
+    pip3 install pyexpander
 
-ARG EXTRA_VARIANTS
-ARG CUDA_VARIANT
-RUN sed \
-    -e "s|packages:|packages:\n  zfp:\n    variants: ${CUDA_VARIANT} +shared|" \
-    -e "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \
-    -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \
-    -i /etc/spack/packages.yaml
+COPY packages.yaml.in packages.yaml.in
+COPY specs.yaml.in specs.yaml.in
 
+ARG DEVICE_FLAVOR=cpu
+ARG MPI_VARIANT=~mpi
 ARG MPI_FLAVOR=openmpi
-RUN sed "s|^\(\s\+\)all:.*|&\n\1  providers:\n\1    mpi: [${MPI_FLAVOR}]|" \
-      -i /etc/spack/packages.yaml && \
-      cat /etc/spack/packages.yaml
+
+RUN expander.py --eval "device_flavor=\"${DEVICE_FLAVOR}\"" specs.yaml.in > specs.yaml && \
+    expander.py --eval "device_flavor=\"${DEVICE_FLAVOR}\";mpi_variant=\"${MPI_VARIANT}\";mpi_flavor=\"${MPI_FLAVOR}\"" packages.yaml.in > packages.yaml
+
+FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base
+
+COPY --from=builder /root/packages.yaml /etc/spack/packages.yaml
+COPY --from=builder /root/specs.yaml /tmp/specs.yaml
 
 # Build dependencies
 ARG COMPILER_SPACK_ID
@@ -25,40 +30,25 @@ RUN . /etc/profile.d/modules.sh && \
     spack spec --reuse adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
     spack install \
       --no-checksum \
-      --reuse \
       --fail-fast \
       -v \
-      -j$(grep -c '^processor' /proc/cpuinfo) \
+      -j$((2 * $(grep -c '^processor' /proc/cpuinfo))) \
       --only dependencies \
       adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
-    spack spec --reuse libsodium%${COMPILER_SPACK_ID} && \
-    spack install \
-      --no-checksum \
-      --reuse \
-      --fail-fast \
-      -v \
-      -j$(grep -c '^processor' /proc/cpuinfo) \
-      libsodium%${COMPILER_SPACK_ID} && \
     spack clean -a
 
 # Setup modules
 RUN . /opt/spack/share/spack/setup-env.sh && \
-    spack env create --without-view adios2-ci && \
-    spack -e adios2-ci config add concretizer:unify:true && \
-    spack -e adios2-ci add $(spack find --format "/{hash}") && \
-    spack -e adios2-ci rm cuda && \
-    spack -e adios2-ci install && \
-    spack env activate adios2-ci && \
-    spack env deactivate && \
-    rm -rf /root/.spack && \
-    spack -e adios2-ci env loads
+    spack env create adios2-ci /tmp/specs.yaml && \
+    spack -e adios2-ci add $(spack find --implicit --format "/{hash}") && \
+    spack -e adios2-ci concretize --force --fresh && \
+    spack -e adios2-ci install \
+      --no-checksum \
+      --fail-fast \
+      -v \
+      -j$((2 * $(grep -c '^processor' /proc/cpuinfo)))
 
 # Setup default login environment
-RUN . /opt/spack/share/spack/setup-env.sh && \
-    echo 'source /opt/spack/share/spack/setup-env.sh' >> /etc/profile.d/zz-adios2-ci-env.sh && \
-    echo "module use ${SPACK_ROOT}/share/spack/modules/$(spack debug report | grep -Po 'Platform:\*\*\s\K.*')" >> /etc/profile.d/zz-adios2-ci-env.sh && \
-    echo 'source ${SPACK_ROOT}/var/spack/environments/adios2-ci/loads' >> /etc/profile.d/zz-adios2-ci-env.sh
-
-# Enable CUDA in Spack if enabled
-COPY spack-external-cuda-addon.bash /root/spack-external-cuda-addon.bash
-RUN test x"${CUDA_VARIANT}" == x"+cuda" && . /root/spack-external-cuda-addon.bash
+RUN . /etc/profile.d/modules.sh && \
+    echo "source /opt/spack/share/spack/setup-env.sh" >> /etc/profile.d/zz-adios2-ci-env.sh && \
+    echo "spack env activate adios2-ci" >> /etc/profile.d/zz-adios2-ci-env.sh

scripts/ci/images-v2/build-cuda.sh

@@ -1,38 +1,19 @@
 #!/bin/bash
-set +e
+set -ex
 
 BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE}))
 source "${BASE_DIR}/build-functions.sh"
 
-message1 "Building ci-spack-el8 base image"
-if ! build_squash \
-    nvidia/cuda:11.7.0-devel-rockylinux8 \
-    ornladios/adios2:ci-spack-el8-cuda-base \
-    Dockerfile.ci-spack-el8-base \
-    "--build-arg=BASE_IMAGE=nvidia/cuda:11.7.0-devel-rockylinux8"
-then
-  echo "Error: Failed to build ci-spack-el8 base image"
-  exit 3
-fi
-
-message1 "Building ci-spack-el8 gcc8 base image"
-if ! build_squash \
-    ornladios/adios2:ci-spack-el8-cuda-base \
-    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
-    Dockerfile.ci-spack-el8-gcc8-base \
-    "--build-arg=BASE_IMAGE=ornladios/adios2:ci-spack-el8-cuda-base"
-then
-  echo "Error: Failed to build ci-spack-el8 leaf image"
-  exit 3
-fi
-
 message1 "Building ci-spack-el8 leaf image"
-build_conf=cuda-gcc8
+build_conf=gcc8
 if ! build_squash \
-    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
+    ornladios/adios2:ci-spack-el8-gcc8-base \
     ornladios/adios2:ci-spack-el8-cuda-serial \
     Dockerfile.ci-spack-el8-leaf \
-    "--build-arg=COMPILER_IMG_BASE=${build_conf} --build-arg=COMPILER_SPACK_ID=gcc --build-arg=CUDA_VARIANT=+cuda"
+    "--build-arg=COMPILER_IMG_BASE=${build_conf}
+     --build-arg=COMPILER_SPACK_ID=gcc
+     --build-arg=DEVICE_FLAVOR=CUDA
+     --build-arg=MPI_VARIANT=~mpi"
 then
   echo "Error: Failed to build ci-spack-el8 leaf image"
   exit 3

scripts/ci/images-v2/build-gcc.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set +e
+set -x
 
 BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE}))
 source "${BASE_DIR}/build-functions.sh"
@@ -11,7 +12,9 @@ do
     ornladios/adios2:ci-spack-el8-gcc${ver}-base \
     ornladios/adios2:ci-spack-el8-gcc${ver}-serial \
     Dockerfile.ci-spack-el8-leaf \
-    "--build-arg COMPILER=gcc${ver} --build-arg EXTRA_VARIANTS=~mpi"
+    "--build-arg COMPILER_IMG_BASE=gcc${ver}
+     --build-arg COMPILER_SPACK_ID=gcc
+     --build-arg=MPI_VARIANT=~mpi"
   then
     echo "Error: Failed to build gcc${ver}-serial image"
     exit 3
@@ -22,7 +25,9 @@ do
     ornladios/adios2:ci-spack-el8-gcc${ver}-base \
     ornladios/adios2:ci-spack-el8-gcc${ver}-mpi \
     Dockerfile.ci-spack-el8-leaf \
-    "--build-arg COMPILER=gcc${ver} --build-arg EXTRA_VARIANTS=+mpi"
+    "--build-arg COMPILER_IMG_BASE=gcc${ver}
+     --build-arg COMPILER_SPACK_ID=gcc
+     --build-arg=MPI_VARIANT=+mpi"
   then
     echo "Error: Failed to build gcc${ver}-mpi image"
     exit 3

scripts/ci/images-v2/build-mpich.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -ex
+
+BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE}))
+source "${BASE_DIR}/build-functions.sh"
+
+message1 "Building ci-spack-el8 leaf image"
+build_conf=gcc10
+if ! build_squash \
+    ornladios/adios2:ci-spack-el8-gcc10-base \
+    ornladios/adios2:ci-spack-el8-gcc10-mpich \
+    Dockerfile.ci-spack-el8-leaf \
+    "--build-arg=COMPILER_IMG_BASE=${build_conf}
+     --build-arg=COMPILER_SPACK_ID=gcc
+     --build-arg=MPI_FLAVOR=mpich
+     --build-arg=MPI_VARIANT=+mpi"
+then
+  echo "Error: Failed to build ci-spack-el8 leaf image"
+  exit 3
+fi

scripts/ci/images-v2/modules.yaml

@@ -1,9 +1,10 @@
 modules:
-  enable:
-  - tcl
-  tcl:
-    hash_length: 0
-    all:
-      autoload: 'direct'
-    projections:
-      all: '{name}/{version}'
+  default:
+    enable:
+    - tcl
+    tcl:
+      hash_length: 0
+      all:
+        autoload: 'direct'
+      projections:
+        all: '{name}/{version}'

scripts/ci/images-v2/packages.yaml.in

@@ -0,0 +1,16 @@
+packages:
+  all:
+    target: [haswell]
+    providers:
+      mpi: [$(mpi_flavor)]
+  adios2:
+    variants: $(mpi_variant)+blosc+bzip2+zfp+sz+png+sst+dataman+ssc+hdf5+python+fortran
+  cmake:
+    version: [3.22.4]
+  mpich:
+    version: [3.3.2]
+    variants: device=ch3 netmod=tcp
+$if(device_flavor == "CUDA")\
+  zfp:
+    variants: +cuda+shared
+$endif\

scripts/ci/images-v2/specs.yaml.in

@@ -0,0 +1,13 @@
+$# Run this with pyexpander --eval "flavor=xxyzz"
+$default(compiler_spack_id="gcc",device_flavor="BASE")\
+spack:
+  specs:
+    - libsodium%$(compiler_spack_id)
+$if(device_flavor == "CUDA")\
+    - kokkos@3.6.01%gcc+cuda+wrapper cuda_arch=52
+$endif\
+  view: true
+  config:
+    checksum: false
+  concretizer:
+    unify: false