Merge branch '2023.06-software.eessi.io' into 2023.06-software.eessi.…

…io_ci-missing-installations

.github/workflows/test_eessi_container_script.yml

@@ -45,7 +45,7 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
                 outfile=out_listrepos.txt
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
               # test use of --list-repos with custom repos.cfg
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
@@ -57,7 +57,7 @@ jobs:
                 echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
                 echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
                 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
@@ -90,15 +90,15 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
                 outfile=out_readwrite.txt
                 fn="test_${RANDOM}.txt"
-                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
                 ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
-                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
@@ -120,12 +120,12 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
                 outfile=out_save.txt
                 fn="test_${RANDOM}.txt"
-                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                test_cmd="touch /cvmfs/software.eessi.io/${fn}"
                 ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 rm -f ${outfile}
 
-                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 

.github/workflows/tests_archdetect.yml

@@ -13,24 +13,59 @@ jobs:
           - x86_64/intel/skylake_avx512/archspec-linux-6132
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
-          - ppc64le/power9le/unknown-power9le
-          - aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra
-          - aarch64/arm/neoverse-n1/AWS-awslinux-graviton2
-          - aarch64/arm/neoverse-v1/AWS-awslinux-graviton3
+          - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
+          - aarch64/neoverse_n1/AWS-awslinux-graviton2
+          - aarch64/neoverse_v1/AWS-awslinux-graviton3
+          # commented out since these targets are currently not supported in software.eessi.io repo
+          # (and some tests assume that the corresponding subdirectory in software layer is there)
+          # - ppc64le/power9le/unknown-power9le
       fail-fast: false
     steps:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
+    - name: Mount EESSI CernVM-FS pilot repository
+      uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1
+      with:
+          cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+          cvmfs_http_proxy: DIRECT
+          cvmfs_repositories: software.eessi.io
+
     - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
           export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*}
           export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo
+          # check that printing of best match works correctly
           CPU_ARCH=$(./init/eessi_archdetect.sh cpupath)
           if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then
-              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2
+              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH"
           else
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2
               exit 1
           fi
+          # check that $EESSI_SOFTWARE_SUBDIR_OVERRIDE is honored
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE='dummy/cpu'
+          CPU_ARCH=$(./init/eessi_archdetect.sh cpupath)
+          if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+              echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE PASSED"
+          else
+              echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE FAILED" >&2
+              exit 1
+          fi
+          unset EESSI_SOFTWARE_SUBDIR_OVERRIDE
+          # check that printing of all matches works correctly (-a option for cpupath action)
+          CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath)
+          if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then
+              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES"
+          else
+              echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2
+              exit 1
+          fi
+          # Check all those architectures actually exist (if this EESSI version has been populated already)
+          if [ -d ${EESSI_PREFIX}/software/linux ]; then
+            for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
+                # Search all EESSI versions as we may drop support at some point
+                ls -d ${EESSI_PREFIX}/software/linux/${dir}
+            done
+          fi

.github/workflows/tests_scripts.yml

@@ -5,7 +5,7 @@ on:
     paths:
       - build_container.sh
       - create_directory_tarballs.sh
-      - EESSI-pilot-install-software.sh
+      - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
@@ -16,7 +16,7 @@ on:
     paths:
       - build_container.sh
       - create_directory_tarballs.sh
-      - EESSI-pilot-install-software.sh
+      - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
@@ -40,7 +40,7 @@ jobs:
         # bind current directory into container as /software-layer
         export SINGULARITY_BIND="${PWD}:/software-layer"
 
-        # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06,
+        # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06,
         # since Python in compat layer is Python 3.11.x;
         # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version
         for EB_VERSION in '4.6.0'; do
@@ -79,20 +79,18 @@ jobs:
 
     - name: test install_software_layer.sh script
       run: |
-          # scripts need to be copied to /tmp,
-          # since install_software_layer.sh must be accessible from within build container
-          cp -a * /tmp/
-          cd /tmp
+          # bind current directory into container as /software-layer
+          export SINGULARITY_BIND="${PWD}:/software-layer"
           # force using x86_64/generic, to avoid triggering an installation from scratch
-          sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+          sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh
+          ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh
 
     - name: test create_directory_tarballs.sh script
       run: |
+          # bind current directory into container as /software-layer
+          export SINGULARITY_BIND="${PWD}:/software-layer"
           # scripts need to be copied to /tmp,
           # since create_directory_tarballs.sh must be accessible from within build container
-          cp -a * /tmp/
-          cd /tmp
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06
+          ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06
           # check if tarballs have been produced
           ls -l *.tar.gz

EESSI-pilot-install-software.sh → EESSI-install-software.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to install EESSI pilot software stack (version set through init/eessi_defaults)
+# Script to install EESSI software stack (version set through init/eessi_defaults)
 
 # see example parsing of command line arguments at
 #   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
@@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then
     export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
 fi
 
-${EB} --show-config
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
@@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[
     # load EasyBuild module (will be installed if it's not available yet)
     source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
+    ${EB} --show-config
+
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
     if [ -f ${easystack_file} ]; then

bot/build.sh

@@ -128,12 +128,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
-# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
-echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
@@ -210,9 +210,9 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 timestamp=$(date +%s)
-# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
+# to set EESSI_VERSION we need to source init/eessi_defaults now
 source init/eessi_defaults
-export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
 
 # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
 # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
@@ -221,8 +221,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

bot/check-test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Dummy script that only creates test result file for the bot, without actually checking anything
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hoste (HPC-UGent)
+#
+# license: GPLv2
+#
+job_dir=${PWD}
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+
+echo "[TEST]" > ${job_test_result_file}
+echo "comment_description = <em>(no tests yet)</em>" >> ${job_test_result_file}
+echo "status = SUCCESS" >> ${job_test_result_file}
+
+exit 0

bot/inspect.sh

@@ -195,12 +195,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
-# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
-echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
@@ -260,11 +260,11 @@ base_dir=$(dirname $(realpath $0))
 # TODO better use script from tarball???
 source ${base_dir}/../init/eessi_defaults
 
-if [ -z $EESSI_PILOT_VERSION ]; then
-    echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
+if [ -z $EESSI_VERSION ]; then
+    echo "ERROR: \$EESSI_VERSION must be set!" >&2
     exit 1
 fi
-EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)"
+EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)"
 
 # NOTE The below requires access to the CVMFS repository. We could make a first
 # test run with a container. For now we skip the test.
@@ -295,8 +295,8 @@ fi
 if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
     echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT}
 fi
-if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then
-    echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT}
+if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
+    echo "export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT}
 fi
 if [ ! -z ${http_proxy} ]; then
     echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT}
@@ -428,7 +428,7 @@ echo "Executing command to start interactive session to inspect build job:"
 # TODO possibly add information on how to init session after the prefix is
 # entered, initialization consists of
 # - environment variable settings (see 'run_in_compat_layer_env.sh')
-# - setup steps run in 'EESSI-pilot-install-software.sh'
+# - setup steps run in 'EESSI-install-software.sh'
 # These initializations are combined into a single script that is executed when
 # the shell in startprefix is started. We set the env variable BASH_ENV here.
 if [[ -z ${run_in_prefix} ]]; then

bot/test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Dummy script, no tests yet
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hoste (HPC-UGent)
+#
+# license: GPLv2
+#
+
+exit 0