From 3fc17467b9665724eeae6651c860fd77d28283d0 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 16 Jun 2023 14:32:20 +0200 Subject: [PATCH 01/25] Allow `archdetect` to print all possible cpu paths Allow fix incorrect generic path --- init/eessi_archdetect.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index d2b6dacf04..97b45bf936 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -VERSION="1.0.0" +VERSION="1.1.0" # Logging LOG_LEVEL="INFO" @@ -105,7 +105,8 @@ cpupath(){ log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'" # Default to generic CPU - local best_arch_match="generic" + local best_arch_match="$machine_type/generic" + local all_arch_matches=$best_arch_match # Iterate over the supported CPU specifications to find the best match for host CPU # Order of the specifications matters, the last one to match will be selected @@ -114,22 +115,28 @@ cpupath(){ if [ "${cpu_vendor}x" == "${arch_spec[1]}x" ]; then # each flag in this CPU specification must be found in the list of flags of the host check_allinfirst "${cpu_flags[*]}" ${arch_spec[2]} && best_arch_match=${arch_spec[0]} && \ - log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" + all_arch_matches="$best_arch_match:$all_arch_matches" && \ + log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" fi done log "INFO" "cpupath: best match for host CPU: $best_arch_match" - echo "$best_arch_match" + if [ "allx" == "${1}x" ]; then + echo "$all_arch_matches" + else + echo "$best_arch_match" + fi } # Parse command line arguments -USAGE="Usage: eessi_archdetect.sh [-h][-d] " +USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] " -while getopts 'hdv' OPTION; do +while getopts 'hdva' OPTION; do case "$OPTION" in h) echo "$USAGE"; exit 0;; d) LOG_LEVEL="DEBUG";; v) echo "eessi_archdetect.sh v$VERSION"; exit 0;; + a) all="all";; ?) echo "$USAGE"; exit 1;; esac done @@ -138,6 +145,6 @@ shift "$(($OPTIND -1))" ARGUMENT=${1:-none} case "$ARGUMENT" in - "cpupath") cpupath; exit;; - *) echo "$USAGE"; log "ERROR" "Missing argument";; + "cpupath") cpupath $all; exit;; + *) echo "$USAGE"; log "ERROR" "Missing argument (possible actions: 'cpupath')";; esac From 551d0082bcb15fd0cb90cf958eb4280b03dc2a58 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 16 Jun 2023 15:44:08 +0200 Subject: [PATCH 02/25] Add tests --- .github/workflows/tests_archdetect.yml | 7 +++++++ .../arm/neoverse-n1/AWS-awslinux-graviton2.all.output | 1 + .../arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output | 1 + .../arm/neoverse-v1/AWS-awslinux-graviton3.all.output | 1 + .../ppc64le/power9le/unknown-power9le.all.output | 1 + .../x86_64/amd/zen2/Azure-CentOS7-7V12.all.output | 1 + .../x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output | 1 + .../intel/haswell/archspec-linux-E5-2680-v3.all.output | 1 + .../intel/skylake_avx512/archspec-linux-6132.all.output | 1 + 9 files changed, 15 insertions(+) create mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output create mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output create mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output create mode 100644 tests/archdetect/ppc64le/power9le/unknown-power9le.all.output create mode 100644 tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output create mode 100644 tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output create mode 100644 tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output create mode 100644 tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 618f6eb142..cabbaedb6d 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -34,3 +34,10 @@ jobs: echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2 exit 1 fi + CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath) + if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" >&2 + else + echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 + exit 1 + fi diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output new file mode 100644 index 0000000000..4e4f20c1de --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output new file mode 100644 index 0000000000..4e4f20c1de --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output new file mode 100644 index 0000000000..61f4d892f0 --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse-v1:aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output new file mode 100644 index 0000000000..7ecf79d0a7 --- /dev/null +++ b/tests/archdetect/ppc64le/power9le/unknown-power9le.all.output @@ -0,0 +1 @@ +ppc64le/power9le:ppc64le/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output new file mode 100644 index 0000000000..180de26f0e --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen2/Azure-CentOS7-7V12.all.output @@ -0,0 +1 @@ +x86_64/amd/zen2:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output new file mode 100644 index 0000000000..798a0aa565 --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen3/Azure-CentOS7-7V73X.all.output @@ -0,0 +1 @@ +x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output new file mode 100644 index 0000000000..a047dd42cc --- /dev/null +++ b/tests/archdetect/x86_64/intel/haswell/archspec-linux-E5-2680-v3.all.output @@ -0,0 +1 @@ +x86_64/intel/haswell:x86_64/generic \ No newline at end of file diff --git a/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output new file mode 100644 index 0000000000..c9fa524ea6 --- /dev/null +++ b/tests/archdetect/x86_64/intel/skylake_avx512/archspec-linux-6132.all.output @@ -0,0 +1 @@ +x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic \ No newline at end of file From 7db0616a0c78f40864507d77085817af6eb7028a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 1 Sep 2023 11:58:10 +0200 Subject: [PATCH 03/25] Remove need to pass argument to function --- init/eessi_archdetect.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 97b45bf936..58dd99eb6b 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -3,6 +3,8 @@ VERSION="1.1.0" # Logging LOG_LEVEL="INFO" +# Default result type is a best match +CPUPATH_RESULT="best" timestamp () { date "+%Y-%m-%d %H:%M:%S" @@ -120,10 +122,11 @@ cpupath(){ fi done - log "INFO" "cpupath: best match for host CPU: $best_arch_match" - if [ "allx" == "${1}x" ]; then + if [ "allx" == "${CPUPATH_RESULT}x" ]; then + log "INFO" "cpupath: all matches for host CPU: $all_arch_matches" echo "$all_arch_matches" else + log "INFO" "cpupath: best match for host CPU: $best_arch_match" echo "$best_arch_match" fi } @@ -136,7 +139,7 @@ while getopts 'hdva' OPTION; do h) echo "$USAGE"; exit 0;; d) LOG_LEVEL="DEBUG";; v) echo "eessi_archdetect.sh v$VERSION"; exit 0;; - a) all="all";; + a) CPUPATH_RESULT="all";; ?) echo "$USAGE"; exit 1;; esac done @@ -145,6 +148,6 @@ shift "$(($OPTIND -1))" ARGUMENT=${1:-none} case "$ARGUMENT" in - "cpupath") cpupath $all; exit;; + "cpupath") cpupath; exit;; *) echo "$USAGE"; log "ERROR" "Missing argument (possible actions: 'cpupath')";; esac From b81b729f9350d10dee15a3d561417c688258e99d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 16:40:32 +0200 Subject: [PATCH 04/25] Make sure all the directories actually exist --- .github/workflows/tests_archdetect.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index cabbaedb6d..676a7a8c4d 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -23,6 +23,9 @@ jobs: uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: test eessi_archdetect.sh + uses: eessi/github-action-eessi@main + with: + eessi_stack_version: '2023.06' run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*} @@ -41,3 +44,7 @@ jobs: echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 exit 1 fi + # Check all those architectures exist + for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do + [ -d "$EESSI_PREFIX/software/linux/$dir" ] + done From 739a92da2dfabe70da52ae433c5152337effa494 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 16:44:16 +0200 Subject: [PATCH 05/25] Fix steps in action --- .github/workflows/tests_archdetect.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 676a7a8c4d..92907607c4 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -21,11 +21,10 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: test eessi_archdetect.sh - uses: eessi/github-action-eessi@main + - uses: eessi/github-action-eessi@main with: eessi_stack_version: '2023.06' + - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*} @@ -44,7 +43,7 @@ jobs: echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 exit 1 fi - # Check all those architectures exist + # Check all those architectures actually exist for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do [ -d "$EESSI_PREFIX/software/linux/$dir" ] done From 45dc7dadd07ec5bdc4bb8f29b43d3eb75f5f9c34 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 16:57:17 +0200 Subject: [PATCH 06/25] Fix ARM detection for archdetect --- .github/workflows/tests_archdetect.yml | 6 +++--- init/arch_specs/eessi_arch_arm.spec | 8 ++++---- .../arm/neoverse-n1/AWS-awslinux-graviton2.all.output | 1 - .../aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output | 1 - .../arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output | 1 - .../aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output | 1 - .../arm/neoverse-v1/AWS-awslinux-graviton3.all.output | 1 - .../aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output | 1 - .../arm/neoverse_n1/AWS-awslinux-graviton2.all.output | 1 + .../AWS-awslinux-graviton2.cpuinfo | 0 .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output | 1 + .../arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output | 1 + .../Azure-Ubuntu20-Altra.cpuinfo | 0 .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output | 1 + .../arm/neoverse_v1/AWS-awslinux-graviton3.all.output | 1 + .../AWS-awslinux-graviton3.cpuinfo | 0 .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output | 1 + 17 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output delete mode 100644 tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output delete mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output delete mode 100644 tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output rename tests/archdetect/aarch64/arm/{neoverse-n1 => neoverse_n1}/AWS-awslinux-graviton2.cpuinfo (100%) create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output rename tests/archdetect/aarch64/arm/{neoverse-n1 => neoverse_n1}/Azure-Ubuntu20-Altra.cpuinfo (100%) create mode 100644 tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output create mode 100644 tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output rename tests/archdetect/aarch64/arm/{neoverse-v1 => neoverse_v1}/AWS-awslinux-graviton3.cpuinfo (100%) create mode 100644 tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 92907607c4..76cc7315c7 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -14,9 +14,9 @@ jobs: - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X - ppc64le/power9le/unknown-power9le - - aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra - - aarch64/arm/neoverse-n1/AWS-awslinux-graviton2 - - aarch64/arm/neoverse-v1/AWS-awslinux-graviton3 + - aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra + - aarch64/arm/neoverse_n1/AWS-awslinux-graviton2 + - aarch64/arm/neoverse_v1/AWS-awslinux-graviton3 fail-fast: false steps: - name: checkout diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 92f32a76d8..97d9cd2edc 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,6 +1,6 @@ # ARM CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"aarch64/arm/neoverse-n1" "ARM" "asimd" # Ampere Altra -"aarch64/arm/neoverse-n1" "" "asimd" # AWS Graviton2 -"aarch64/arm/neoverse-v1" "ARM" "asimd svei8mm" -"aarch64/arm/neoverse-v1" "" "asimd svei8mm" # AWS Graviton3 +"aarch64/arm/neoverse_n1" "ARM" "asimd" # Ampere Altra +"aarch64/arm/neoverse_n1" "" "asimd" # AWS Graviton2 +"aarch64/arm/neoverse_v1" "ARM" "asimd svei8mm" +"aarch64/arm/neoverse_v1" "" "asimd svei8mm" # AWS Graviton3 diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output deleted file mode 100644 index 4e4f20c1de..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.all.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output deleted file mode 100644 index b4dc5e9f1b..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1 diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output deleted file mode 100644 index 4e4f20c1de..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.all.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output deleted file mode 100644 index b4dc5e9f1b..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-n1 diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output deleted file mode 100644 index 61f4d892f0..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.all.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-v1:aarch64/arm/neoverse-n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output deleted file mode 100644 index 20db96d01f..0000000000 --- a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.output +++ /dev/null @@ -1 +0,0 @@ -aarch64/arm/neoverse-v1 diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output new file mode 100644 index 0000000000..e6aee274ae --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-n1/AWS-awslinux-graviton2.cpuinfo rename to tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output new file mode 100644 index 0000000000..c841d54b50 --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_n1 diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output new file mode 100644 index 0000000000..e6aee274ae --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra.cpuinfo rename to tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output new file mode 100644 index 0000000000..c841d54b50 --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_n1 diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output new file mode 100644 index 0000000000..bda6c2e498 --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_v1:aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file diff --git a/tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse-v1/AWS-awslinux-graviton3.cpuinfo rename to tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output new file mode 100644 index 0000000000..0c80cb1efb --- /dev/null +++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output @@ -0,0 +1 @@ +aarch64/arm/neoverse_v1 From 71cf7c95021ab16b65f098b41af853b1534aa62c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:01:13 +0200 Subject: [PATCH 07/25] Fix ARM spec --- init/arch_specs/eessi_arch_arm.spec | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 97d9cd2edc..b5c9275043 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,6 +1,6 @@ # ARM CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"aarch64/arm/neoverse_n1" "ARM" "asimd" # Ampere Altra -"aarch64/arm/neoverse_n1" "" "asimd" # AWS Graviton2 -"aarch64/arm/neoverse_v1" "ARM" "asimd svei8mm" -"aarch64/arm/neoverse_v1" "" "asimd svei8mm" # AWS Graviton3 +"aarch64/neoverse_n1" "ARM" "asimd" # Ampere Altra +"aarch64/neoverse_n1" "" "asimd" # AWS Graviton2 +"aarch64/neoverse_v1" "ARM" "asimd svei8mm" +"aarch64/neoverse_v1" "" "asimd svei8mm" # AWS Graviton3 From 98a95feaaee00046ec233ca92f37e7862d6b3035 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:05:15 +0200 Subject: [PATCH 08/25] Also fix archdetect tests --- .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output | 2 +- .../aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output | 2 +- .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output | 2 +- .../aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output | 2 +- .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output | 2 +- .../aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output index e6aee274ae..340aaa5d02 100644 --- a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output +++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output @@ -1 +1 @@ -aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file +aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output index c841d54b50..a9bd49c75c 100644 --- a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output +++ b/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output @@ -1 +1 @@ -aarch64/arm/neoverse_n1 +aarch64/neoverse_n1 diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output index e6aee274ae..340aaa5d02 100644 --- a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output +++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output @@ -1 +1 @@ -aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file +aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output index c841d54b50..a9bd49c75c 100644 --- a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output +++ b/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output @@ -1 +1 @@ -aarch64/arm/neoverse_n1 +aarch64/neoverse_n1 diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output index bda6c2e498..920d5f9996 100644 --- a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output +++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output @@ -1 +1 @@ -aarch64/arm/neoverse_v1:aarch64/arm/neoverse_n1:aarch64/generic \ No newline at end of file +aarch64/neoverse_v1:aarch64/neoverse_n1:aarch64/generic diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output index 0c80cb1efb..a8e072a9c6 100644 --- a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output +++ b/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output @@ -1 +1 @@ -aarch64/arm/neoverse_v1 +aarch64/neoverse_v1 From ef1a53aff3b22b120a0405aaaa0521bb6426f219 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:20:00 +0200 Subject: [PATCH 09/25] Make tests flexible in terms of EESSI version --- .github/workflows/tests_archdetect.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 76cc7315c7..b6fc6434b2 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -21,9 +21,7 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: eessi/github-action-eessi@main - with: - eessi_stack_version: '2023.06' + - uses: eessi/github-action-eessi@v2 - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} @@ -45,5 +43,6 @@ jobs: fi # Check all those architectures actually exist for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do - [ -d "$EESSI_PREFIX/software/linux/$dir" ] + # Search all EESSI versions as we may drop support at some point + ls -d "$EESSI_PREFIX/../*/software/linux/$dir" done From 5665fad15dea939751ad6eb17286a4670353f973 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:28:38 +0200 Subject: [PATCH 10/25] Fix hash of EESSI action --- .github/workflows/tests_archdetect.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index b6fc6434b2..f5781ef2a7 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -21,7 +21,9 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: eessi/github-action-eessi@v2 + - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + with: + eessi_stack_version: '2023.06' - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} From 07bfbb1d9b14bdb971d82b7a6d61c7e9818d8946 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:32:41 +0200 Subject: [PATCH 11/25] Careful with quotes --- .github/workflows/tests_archdetect.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index f5781ef2a7..a794e5ebc4 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -21,7 +21,8 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + - name: Enable EESSI + uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 with: eessi_stack_version: '2023.06' - name: test eessi_archdetect.sh @@ -46,5 +47,5 @@ jobs: # Check all those architectures actually exist for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do # Search all EESSI versions as we may drop support at some point - ls -d "$EESSI_PREFIX/../*/software/linux/$dir" + ls -d "$EESSI_PREFIX"/../*/software/linux/"$dir" done From b10415b16834b15f17a49ade9391d84f1fc2d469 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:43:46 +0200 Subject: [PATCH 12/25] Also drop arm subdir from tests --- .github/workflows/tests_archdetect.yml | 6 +++--- .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.all.output | 0 .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.cpuinfo | 0 .../{arm => }/neoverse_n1/AWS-awslinux-graviton2.output | 0 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.all.output | 0 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo | 0 .../{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.output | 0 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.all.output | 0 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.cpuinfo | 0 .../{arm => }/neoverse_v1/AWS-awslinux-graviton3.output | 0 10 files changed, 3 insertions(+), 3 deletions(-) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.all.output (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.cpuinfo (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/AWS-awslinux-graviton2.output (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.all.output (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_n1/Azure-Ubuntu20-Altra.output (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.all.output (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.cpuinfo (100%) rename tests/archdetect/aarch64/{arm => }/neoverse_v1/AWS-awslinux-graviton3.output (100%) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index a794e5ebc4..61bf0e73fd 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -14,9 +14,9 @@ jobs: - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X - ppc64le/power9le/unknown-power9le - - aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra - - aarch64/arm/neoverse_n1/AWS-awslinux-graviton2 - - aarch64/arm/neoverse_v1/AWS-awslinux-graviton3 + - aarch64/neoverse_n1/Azure-Ubuntu20-Altra + - aarch64/neoverse_n1/AWS-awslinux-graviton2 + - aarch64/neoverse_v1/AWS-awslinux-graviton3 fail-fast: false steps: - name: checkout diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.all.output rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.all.output diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.cpuinfo rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output b/tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/AWS-awslinux-graviton2.output rename to tests/archdetect/aarch64/neoverse_n1/AWS-awslinux-graviton2.output diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.all.output rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.all.output diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output b/tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_n1/Azure-Ubuntu20-Altra.output rename to tests/archdetect/aarch64/neoverse_n1/Azure-Ubuntu20-Altra.output diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.all.output rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.all.output diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.cpuinfo rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.cpuinfo diff --git a/tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output b/tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output similarity index 100% rename from tests/archdetect/aarch64/arm/neoverse_v1/AWS-awslinux-graviton3.output rename to tests/archdetect/aarch64/neoverse_v1/AWS-awslinux-graviton3.output From 409ba53ba360268449cffb3de59a91700eb1f78f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 3 Sep 2023 17:45:24 +0200 Subject: [PATCH 13/25] Remove EESSI version selection --- .github/workflows/tests_archdetect.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 61bf0e73fd..37338693c5 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -23,8 +23,6 @@ jobs: uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Enable EESSI uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 - with: - eessi_stack_version: '2023.06' - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} From 92b764998fc46b0906b20de7346d0044248281b4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Oct 2023 10:59:49 +0200 Subject: [PATCH 14/25] remove bot configuration (main) --- bot/bot-eessi-aws-citc.cfg | 151 ------------------------------------- 1 file changed, 151 deletions(-) delete mode 100644 bot/bot-eessi-aws-citc.cfg diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg deleted file mode 100644 index 5b3ad34612..0000000000 --- a/bot/bot-eessi-aws-citc.cfg +++ /dev/null @@ -1,151 +0,0 @@ -# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 - -[github] -# replace '123456' with the ID of your GitHub App -app_id = 281041 - -# a short (!) name for your app instance that can be used for example -# when adding/updating a comment to a PR -# (!) a short yet descriptive name is preferred because it appears in -# comments to the PR -# for example, the name could include the name of the cluster the bot -# runs on and the username which runs the bot -# NOTE avoid putting an actual username here as it will be visible on -# potentially publicly accessible GitHub pages. -app_name = eessi-bot-citc-aws - -# replace '12345678' with the ID of the installation of your GitHub App -# (can be derived by creating an event and then checking for the list -# of sent events and its payload either via the Smee channel's web page -# or via the Advanced section of your GitHub App on github.com) -installation_id = 33078935 - -# path to the private key that was generated when the GitHub App was registered -private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem - - -[buildenv] -# name of the job script used for building an EESSI stack -build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm - -# The container_cachedir may be used to reuse downloaded container image files -# across jobs. Thus, jobs can more quickly launch containers. -container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir - -# it may happen that we need to customize some CVMFS configuration -# the value of cvmfs_customizations is a dictionary which maps a file -# name to an entry that needs to be added to that file -cvmfs_customizations = {} - -# if compute nodes have no internet connection, we need to set http(s)_proxy -# or commands such as pip3 cannot download software from package repositories -# for example, the temporary EasyBuild is installed via pip3 first -# http_proxy = http://PROXY_DNS:3128/ -# https_proxy = http://PROXY_DNS:3128/ - -# directory under which the bot prepares directories per job -# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR -jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# configure environment -# list of comma-separated modules to be loaded by build_job_script -# useful/needed if some tool is not provided as system-wide package -# (read by bot and handed over to build_job_script via parameter -# --load-modules) -load_modules = - -# PATH to temporary directory on build node ... ends up being used for -# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI -# escaping variables with '\' delays expansion to the start of the -# build_job_script; this can be used for referencing environment -# variables that are only set inside a Slurm job -local_tmp = /tmp/$USER/EESSI - -# parameters to be added to all job submissions -# NOTE do not quote parameter string. Quotes are retained when reading in config and -# then the whole 'string' is recognised as a single parameter. -# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs -# to be initialised as if it is for a login shell. -# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed -slurm_params = --hold --time=24:0:0 - -# full path to the job submission command -submit_command = /usr/bin/sbatch - -# which GH account has the permission to trigger the build (by setting -# the label 'bot:build' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the build -# value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge - -[architecturetargets] -# defines both for which architectures the bot will build -# and what submission parameters shall be used -# medium instances (8 cores, 16GB RAM) -#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} -# larger instances (16 cores, 32GB RAM) -arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} - -[repo_targets] -# defines for which repository a arch_target should be build for -# -# only building for repository EESSI-pilot -repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} - -# points to definition of repositories (default EESSI-pilot defined by build container) -repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles - -# configuration for event handler which receives events from a GitHub repository. -[event_handler] -# path to the log file to log messages for event handler -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log - - -[job_manager] -# path to the log file to log messages for job manager -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log - -# directory where job manager stores information about jobs to be tracked -# e.g. as symbolic link JOBID -> directory to job -job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# full path to the job status checking command -poll_command = /usr/bin/squeue - -# polling interval in seconds -poll_interval = 60 - -# full path to the command for manipulating existing jobs -scontrol_command = /usr/bin/scontrol - -[deploycfg] -# script for uploading built software packages -tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging - -# URL to S3/minio bucket -# if attribute is set, bucket_base will be constructed as follows -# bucket_base=${endpoint_url}/${bucket_name} -# otherwise, bucket_base will be constructed as follows -# bucket_base=https://${bucket_name}.s3.amazonaws.com -# - The former variant is used for non AWS S3 services, eg, minio, or when -# the bucket name is not provided in the hostname (see latter case). -# - The latter variant is used for AWS S3 services. -#endpoint_url = URL_TO_S3_SERVER - -# bucket name -bucket_name = eessi-staging - -# upload policy: defines what policy is used for uploading built artefacts -# to an S3 bucket -# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) -# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) -# only upload the latest built artefact -# 'once' : only once upload any built artefact for the build target -# 'none' : do not upload any built artefacts -upload_policy = once - -# which GH account has the permission to trigger the deployment (by setting -# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the deployment -# value can be a space delimited list of GH accounts -deploy_permission = boegel trz42 bedroge From a095a9d02418b42f882d5c468f2eb6fda8cf3e5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:36:59 +0100 Subject: [PATCH 15/25] get rid of 'pilot' terminology in scripts --- .github/workflows/tests_scripts.yml | 8 ++++---- ...l-software.sh => EESSI-install-software.sh | 6 +++--- bot/build.sh | 16 +++++++-------- bot/inspect.sh | 20 +++++++++---------- install_software_layer.sh | 2 +- run_in_compat_layer_env.sh | 12 +++++------ 6 files changed, 32 insertions(+), 32 deletions(-) rename EESSI-pilot-install-software.sh => EESSI-install-software.sh (98%) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 74e2ebcffe..589c1458e9 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -5,7 +5,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -16,7 +16,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -40,7 +40,7 @@ jobs: # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" - # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06, + # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, # since Python in compat layer is Python 3.11.x; # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version for EB_VERSION in '4.6.0'; do @@ -84,7 +84,7 @@ jobs: cp -a * /tmp/ cd /tmp # force using x86_64/generic, to avoid triggering an installation from scratch - sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh + sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh - name: test create_directory_tarballs.sh script diff --git a/EESSI-pilot-install-software.sh b/EESSI-install-software.sh similarity index 98% rename from EESSI-pilot-install-software.sh rename to EESSI-install-software.sh index 2dc9d2b325..f6087b3cfe 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-install-software.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to install EESSI pilot software stack (version set through init/eessi_defaults) +# Script to install EESSI software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop @@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -${EB} --show-config - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge @@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[ # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + ${EB} --show-config + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then diff --git a/bot/build.sh b/bot/build.sh index cf9d69b65f..4af217628e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -128,12 +128,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -210,9 +210,9 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") timestamp=$(date +%s) -# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +# to set EESSI_VERSION we need to source init/eessi_defaults now source init/eessi_defaults -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh @@ -221,8 +221,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/bot/inspect.sh b/bot/inspect.sh index a3b88e5017..9d1fa87e1f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -195,12 +195,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -260,11 +260,11 @@ base_dir=$(dirname $(realpath $0)) # TODO better use script from tarball??? source ${base_dir}/../init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" # NOTE The below requires access to the CVMFS repository. We could make a first # test run with a container. For now we skip the test. @@ -295,8 +295,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT} fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + echo "export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${http_proxy} ]; then echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT} @@ -428,7 +428,7 @@ echo "Executing command to start interactive session to inspect build job:" # TODO possibly add information on how to init session after the prefix is # entered, initialization consists of # - environment variable settings (see 'run_in_compat_layer_env.sh') -# - setup steps run in 'EESSI-pilot-install-software.sh' +# - setup steps run in 'EESSI-install-software.sh' # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. if [[ -z ${run_in_prefix} ]]; then diff --git a/install_software_layer.sh b/install_software_layer.sh index bf3006a4a0..82ca70b73f 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -./run_in_compat_layer_env.sh ./EESSI-pilot-install-software.sh "$@" +./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 12688e2aed..f57c4d0749 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -3,11 +3,11 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 @@ -23,8 +23,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi if [ ! -z ${http_proxy} ]; then INPUT="export http_proxy=${http_proxy}; ${INPUT}" @@ -33,5 +33,5 @@ if [ ! -z ${https_proxy} ]; then INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi -echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 50ec01a558c16c91e185fdc96f973e7cb4e6fee4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:47:48 +0100 Subject: [PATCH 16/25] update eessi_container.sh script for software.eessi.io --- .../workflows/test_eessi_container_script.yml | 16 ++++---- create_tarball.sh | 24 +++++------ eessi_container.sh | 40 ++++++++++--------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 929fb22cec..33122e6ff4 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,7 +45,7 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then @@ -57,7 +57,7 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} @@ -90,15 +90,15 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then outfile=out_readwrite.txt fn="test_${RANDOM}.txt" - echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -120,12 +120,12 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then outfile=out_save.txt fn="test_${RANDOM}.txt" - test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + test_cmd="touch /cvmfs/software.eessi.io/${fn}" ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" diff --git a/create_tarball.sh b/create_tarball.sh index 65f6efc2dc..8510caebf1 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -5,11 +5,11 @@ set -e base_dir=$(dirname $(realpath $0)) if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 -pilot_version=$2 +eessi_version=$2 cpu_arch_subdir=$3 target_tgz=$4 @@ -20,7 +20,7 @@ os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 @@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' # using EasyBuild's standard module naming scheme) match the name of the @@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ | grep -v '/\.wh\.' >> ${files_list} done fi diff --git a/eessi_container.sh b/eessi_container.sh index 48c4653ba9..bf0294c7bf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -79,7 +79,7 @@ display_help() { echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot via" + echo " repository to use [default: EESSI via" echo " default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" @@ -111,7 +111,7 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" -REPOSITORY="EESSI-pilot" +REPOSITORY="EESSI" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -194,7 +194,7 @@ set -- "${POSITIONAL_ARGS[@]}" if [[ ${LIST_REPOS} -eq 1 ]]; then echo "Listing available repositories with format 'name [source]':" - echo " EESSI-pilot [default]" + echo " EESSI [default]" if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then cfg_load ${EESSI_REPOS_CFG_FILE} sections=$(cfg_sections) @@ -226,7 +226,7 @@ fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" fi @@ -403,7 +403,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then +if [[ "${REPOSITORY}" == "EESSI" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${TOPDIR}/init/eessi_defaults @@ -427,7 +427,7 @@ else # map { local_filepath -> container_filepath } # # repo_name_domain is the domain part of the repo_name, e.g., - # eessi-hpc.org for pilot.eessi-hpc.org + # eessi.io for software.eessi.io # # where config bundle includes the files (-> target location in container) # - default.local -> /etc/cvmfs/default.local @@ -479,7 +479,7 @@ else target=${cfg_file_map[${src}]} BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done - export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} + export EESSI_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) source ${TOPDIR}/init/eessi_defaults @@ -513,10 +513,14 @@ fi # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() + +# always mount cvmfs-config repo (to get access to software.eessi.io) +EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") + if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") export EESSI_FUSE_MOUNTS fi @@ -525,18 +529,18 @@ if [[ "${ACCESS}" == "rw" ]]; then mkdir -p ${EESSI_TMPDIR}/overlay-work # set environment variables for fuse mounts in Singularity container - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_PILOT_WRITABLE_OVERLAY + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_WRITABLE_OVERLAY - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") export EESSI_FUSE_MOUNTS fi From ad6604fd0cf8bd6bd3ded84ec69d041e194332b5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:57:28 +0100 Subject: [PATCH 17/25] remove easystack files used for pilot.eessi-hpc.org --- eessi-2021.06.yml | 53 ------------------------ eessi-2021.12.yml | 69 ------------------------------- eessi-2023.06-eb-4.7.2-2021a.yml | 26 ------------ eessi-2023.06-eb-4.7.2-2021b.yml | 17 -------- eessi-2023.06-eb-4.7.2-2022a.yml | 3 -- eessi-2023.06-eb-4.7.2-2022b.yml | 13 ------ eessi-2023.06-eb-4.7.2-system.yml | 7 ---- eessi-2023.06-eb-4.8.0-2021a.yml | 2 - eessi-2023.06-eb-4.8.0-2021b.yml | 12 ------ eessi-2023.06-eb-4.8.0-system.yml | 10 ----- eessi-2023.06-eb-4.8.1-2021a.yml | 7 ---- eessi-2023.06-eb-4.8.1-2021b.yml | 15 ------- eessi-2023.06-eb-4.8.1-2022a.yml | 12 ------ eessi-2023.06-eb-4.8.1-2023a.yml | 11 ----- eessi-2023.06-eb-4.8.1-system.yml | 9 ---- eessi-2023.06-eb-4.8.2-2021b.yml | 6 --- eessi-2023.06-eb-4.8.2-2022a.yml | 21 ---------- eessi-2023.06-known-issues.yml | 30 -------------- 18 files changed, 323 deletions(-) delete mode 100644 eessi-2021.06.yml delete mode 100644 eessi-2021.12.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2021a.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2021b.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2022a.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2022b.yml delete mode 100644 eessi-2023.06-eb-4.7.2-system.yml delete mode 100644 eessi-2023.06-eb-4.8.0-2021a.yml delete mode 100644 eessi-2023.06-eb-4.8.0-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.0-system.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2021a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2022a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2023a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-system.yml delete mode 100644 eessi-2023.06-eb-4.8.2-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.2-2022a.yml delete mode 100644 eessi-2023.06-known-issues.yml diff --git a/eessi-2021.06.yml b/eessi-2021.06.yml deleted file mode 100644 index 3587827746..0000000000 --- a/eessi-2021.06.yml +++ /dev/null @@ -1,53 +0,0 @@ -software: - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - ReFrame: - toolchains: - SYSTEM: - versions: '3.6.2' - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml deleted file mode 100644 index 210bbb2845..0000000000 --- a/eessi-2021.12.yml +++ /dev/null @@ -1,69 +0,0 @@ -software: - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - gompi-2021a: - versions: ['5.7.1'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - R: - toolchains: - foss-2021a: - versions: '4.1.0' - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - SciPy-bundle: - toolchains: - foss-2021a: - versions: ['2021.05'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - WRF: - toolchains: - foss-2020a: - versions: - '3.9.1.1': - versionsuffix: -dmpar diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml deleted file mode 100644 index 42a42f2236..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ /dev/null @@ -1,26 +0,0 @@ -easyconfigs: - - GCC-10.3.0 - - CMake-3.20.1-GCCcore-10.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2021a - - libpng-1.6.37-GCCcore-10.3.0.eb: - - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - - git-2.32.0-GCCcore-10.3.0-nodocs.eb - - QuantumESPRESSO-6.7-foss-2021a.eb - - GROMACS-2021.3-foss-2021a.eb - - Qt5-5.15.2-GCCcore-10.3.0.eb: - # add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 - options: - from-pr: 18087 - - RapidJSON-1.1.0-GCCcore-10.3.0.eb: - # strip out hardcoded -march=native, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18725 - options: - from-pr: 18725 - - Arrow-6.0.0-foss-2021a.eb: - # fix installation of pyarrow Python bindings - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 - options: - from-pr: 18348 diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml deleted file mode 100644 index 43a0fbd4bd..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ /dev/null @@ -1,17 +0,0 @@ -easyconfigs: - - GCC-11.2.0 - - CMake-3.21.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - CMake-3.22.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - zlib-1.2.11-GCCcore-11.2.0 - - OpenMPI-4.1.1-GCC-11.2.0 - - gompi-2021b - - FlexiBLAS-3.0.4-GCC-11.2.0.eb - - foss-2021b.eb - - git-2.33.1-GCCcore-11.2.0-nodocs.eb - - QuantumESPRESSO-6.8-foss-2021b.eb - - SciPy-bundle-2021.10-foss-2021b - - GROMACS-2021.5-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml deleted file mode 100644 index d40ddff261..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ /dev/null @@ -1,3 +0,0 @@ -easyconfigs: - - GCC-11.3.0 - - OpenMPI-4.1.4-GCC-11.3.0.eb diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml deleted file mode 100644 index b97be34dcf..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ /dev/null @@ -1,13 +0,0 @@ -easyconfigs: - - GCC-12.2.0 - - OpenMPI-4.1.4-GCC-12.2.0 - - CMake-3.24.3-GCCcore-12.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2022b - - netCDF-4.9.0-gompi-2022b.eb: - # use updated CMakeMake easyblock to avoid that -DCMAKE_SKIP_RPATH=ON is used, which breaks the netCDF test step - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3012 - options: - include-easyblocks-from-pr: 3012 - - WRF-4.4.1-foss-2022b-dmpar.eb diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml deleted file mode 100644 index 220fd82e87..0000000000 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - ReFrame-4.2.0.eb: - options: - from-pr: 18320 - - EasyBuild-4.8.0.eb: - options: - from-pr: 18282 diff --git a/eessi-2023.06-eb-4.8.0-2021a.yml b/eessi-2023.06-eb-4.8.0-2021a.yml deleted file mode 100644 index 1adf0572d3..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021a.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - GDAL-3.3.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml deleted file mode 100644 index a2562a4ed1..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - MPFR-4.1.0-GCCcore-11.2.0.eb: - # use patch for MPFR 4.1.0 to fix failing tsprintf test with glibc >= 2.37, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 - options: - from-pr: 18746 - - gnuplot-5.4.2-GCCcore-11.2.0.eb: - # make sure that Lua dependency is correctly picked up, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19261 - options: - from-pr: 19261 - - OpenFOAM-v2112-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml deleted file mode 100644 index 2928f110b0..0000000000 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ /dev/null @@ -1,10 +0,0 @@ -easyconfigs: - - Java-11.eb: - # patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up - # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 - # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 - options: - include-easyblocks-from-pr: 2995 - - EasyBuild-4.8.1.eb: - options: - from-pr: 18761 diff --git a/eessi-2023.06-eb-4.8.1-2021a.yml b/eessi-2023.06-eb-4.8.1-2021a.yml deleted file mode 100644 index 999eb09925..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021a.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - Xvfb-1.20.11-GCCcore-10.3.0.eb: - # fix permission issues for xvfb-run script, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 - options: - from-pr: 18834 - - R-4.1.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml deleted file mode 100644 index 6ee44926ba..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ /dev/null @@ -1,15 +0,0 @@ -easyconfigs: - - Xvfb-1.20.13-GCCcore-11.2.0.eb: - # enable exec permissions for xvfb-run after copying; - # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 - - R-4.2.0-foss-2021b.eb - - PLUMED-2.7.3-foss-2021b.eb: - # the --enable-asmjit is not supported on Aarch64 - options: - from-pr: 19110 - - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: - # TBB and ScaFaCos are optional dependencies when building on Intel arch - options: - from-pr: 19246 diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml deleted file mode 100644 index 8317a40b7a..0000000000 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - OpenBLAS-0.3.20-GCC-11.3.0: - # use easyconfig that includes patch to fix detection of Neoverse V1 CPUs, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18870 - options: - from-pr: 18870 - - foss-2022a - - SciPy-bundle-2022.05-foss-2022a - - BAMM-2.5.0-foss-2022a.eb - - ESPResSo-4.2.1-foss-2022a: - options: - from-pr: 18963 diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml deleted file mode 100644 index c766815934..0000000000 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ /dev/null @@ -1,11 +0,0 @@ -easyconfigs: - - GCC-12.3.0 - - OpenBLAS-0.3.23-GCC-12.3.0: - # add patch to disable flaky DDRGES3 LAPACK test, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18887; - # also pulls in patch to avoid hang when running OpenBLAS test suite, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 - options: - from-pr: 18887 - - foss-2023a - - X11-20230603-GCCcore-12.3.0 diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml deleted file mode 100644 index b0731d2534..0000000000 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ /dev/null @@ -1,9 +0,0 @@ -easyconfigs: - # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - - Java-11.eb - - ReFrame-4.3.3.eb: - options: - from-pr: 18851 - - EasyBuild-4.8.2.eb: - options: - from-pr: 19105 diff --git a/eessi-2023.06-eb-4.8.2-2021b.yml b/eessi-2023.06-eb-4.8.2-2021b.yml deleted file mode 100644 index 00c02adf91..0000000000 --- a/eessi-2023.06-eb-4.8.2-2021b.yml +++ /dev/null @@ -1,6 +0,0 @@ -easyconfigs: - - Pillow-8.3.2-GCCcore-11.2.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.4.3-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml deleted file mode 100644 index ccd3fcd65d..0000000000 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ /dev/null @@ -1,21 +0,0 @@ -easyconfigs: - - casacore-3.5.0-foss-2022a: - options: - from-pr: 19119 - - AOFlagger-3.4.0-foss-2022a: - options: - from-pr: 19119 - - EveryBeam-0.5.2-foss-2022a: - options: - from-pr: 19119 - - DP3-6.0-foss-2022a: - options: - from-pr: 19119 - - WSClean-3.4-foss-2022a: - options: - from-pr: 19119 - - Pillow-9.1.1-GCCcore-11.3.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.5.2-foss-2022a.eb diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml deleted file mode 100644 index ade1c069df..0000000000 --- a/eessi-2023.06-known-issues.yml +++ /dev/null @@ -1,30 +0,0 @@ -- x86_64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/neoverse_v1: - - FFTW-3.3.10-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - FFTW-3.3.10-gompi-2021b: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - OpenBLAS-0.3.18-GCC-11.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.20-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.21-GCC-12.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - - SciPy-bundle-2021.05-foss-2021a: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "2 failing tests (vs 30554 passed) in scipy test suite" - - SciPy-bundle-2021.10-foss-2021b: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" From 34adbe55ca04aa4464eb726f9173bc98aea6af05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:20:36 +0100 Subject: [PATCH 18/25] comment out power9le test for archdetect, since that target is not supported currently in software.eessi.io --- .github/workflows/tests_archdetect.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 37338693c5..01138daad4 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -13,10 +13,12 @@ jobs: - x86_64/intel/skylake_avx512/archspec-linux-6132 - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X - - ppc64le/power9le/unknown-power9le - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 - aarch64/neoverse_v1/AWS-awslinux-graviton3 + # commented out since these targets are currently not supported in software.eessi.io repo + # (and some tests assume that the corresponding subdirectory in software layer is there) + # - ppc64le/power9le/unknown-power9le fail-fast: false steps: - name: checkout From 468c42440ee03678fd5b1188ebdadd4564ab2744 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:36:14 +0100 Subject: [PATCH 19/25] check that archdetect honors $EESSI_SOFTWARE_SUBDIR_OVERRIDE + only check for directories in software-layer if it has been populated already --- .github/workflows/tests_archdetect.yml | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 01138daad4..9cd2775108 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -30,22 +30,36 @@ jobs: export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*} export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo + # check that printing of best match works correctly CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then - echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2 + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" else echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2 exit 1 fi + # check that $EESSI_SOFTWARE_SUBDIR_OVERRIDE is honored + export EESSI_SOFTWARE_SUBDIR_OVERRIDE='dummy/cpu' + CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) + if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE PASSED" + else + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE FAILED" >&2 + exit 1 + fi + unset EESSI_SOFTWARE_SUBDIR_OVERRIDE + # check that printing of all matches works correctly (-a option for cpupath action) CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath) if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then - echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" >&2 + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" else echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 exit 1 fi - # Check all those architectures actually exist - for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do - # Search all EESSI versions as we may drop support at some point - ls -d "$EESSI_PREFIX"/../*/software/linux/"$dir" - done + # Check all those architectures actually exist (if this EESSI version has been populated already) + if [ -d ${EESSI_PREFIX}/software/linux ]; then + for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do + # Search all EESSI versions as we may drop support at some point + ls -d ${EESSI_PREFIX}/software/linux/${dir} + done + fi From 04a71f761e52e6b83b5d879ae22ebdcf862300fb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:38:57 +0100 Subject: [PATCH 20/25] fix picking up on $EESSI_SOFTWARE_SUBDIR_OVERRIDE in archdetect --- init/eessi_archdetect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 58dd99eb6b..58b79b0f2a 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash VERSION="1.1.0" -# Logging -LOG_LEVEL="INFO" +# default log level: only emit warnings or errors +LOG_LEVEL="WARN" # Default result type is a best match CPUPATH_RESULT="best" @@ -69,8 +69,8 @@ check_allinfirst(){ cpupath(){ # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it - log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' " - [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit + log "DEBUG" "cpupath: Override variable set as '$EESSI_SOFTWARE_SUBDIR_OVERRIDE' " + [ $EESSI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} && exit # Identify the best matching CPU architecture from a list of supported specifications for the host CPU # Return the path to the installation files in EESSI of the best matching architecture From d0f26f8f38de507e4d13bc5e055be4de91841764 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:47:50 +0100 Subject: [PATCH 21/25] mount software.eessi.io when testing archdetect --- .github/workflows/tests_archdetect.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 9cd2775108..1e8b830e14 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -23,8 +23,14 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Enable EESSI - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} From 5d532ee7f7933de2c4cab11fe717c33593bf1483 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 17:28:14 +0100 Subject: [PATCH 22/25] stop using build_container.sh script in .github/workflows/tests_scripts.yml, use eessi_container.sh script instead --- .github/workflows/tests_scripts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 589c1458e9..8128a6da94 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -85,7 +85,7 @@ jobs: cd /tmp # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh - ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh + ./eessi_container.sh --mode run --verbose /tmp/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | @@ -93,6 +93,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /tmp/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From 51ce189da2426e1f91acb1462821ecf7550d2d90 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 11:59:44 +0100 Subject: [PATCH 23/25] fix workflow for scripts by bind-mounting repo to /software-layer --- .github/workflows/tests_scripts.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 8128a6da94..a369f4f187 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -79,20 +79,18 @@ jobs: - name: test install_software_layer.sh script run: | - # scripts need to be copied to /tmp, - # since install_software_layer.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh - ./eessi_container.sh --mode run --verbose /tmp/install_software_layer.sh + ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp - ./eessi_container.sh --mode run --verbose /tmp/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From ff615929d282472074531df6b559389f1a06d3d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 15:01:10 +0100 Subject: [PATCH 24/25] add initial easystack file for software.eessi.io to check on EasyBuild v4.8.2 installation --- .../2023.06/eessi-2023.06-eb-4.8.2-system.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml new file mode 100644 index 0000000000..5ce6a65913 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.8.2.eb: + options: + from-pr: 19105 From fd765bbd41691e49d8efcf044fb07e3632049d85 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 21:04:47 +0100 Subject: [PATCH 25/25] add dummy bot/test.sh and bot/check-test.sh scripts --- bot/check-test.sh | 20 ++++++++++++++++++++ bot/test.sh | 13 +++++++++++++ 2 files changed, 33 insertions(+) create mode 100755 bot/check-test.sh create mode 100755 bot/test.sh diff --git a/bot/check-test.sh b/bot/check-test.sh new file mode 100755 index 0000000000..76e0df7f40 --- /dev/null +++ b/bot/check-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Dummy script that only creates test result file for the bot, without actually checking anything +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# +job_dir=${PWD} +job_out="slurm-${SLURM_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM_JOB_ID}.test" + +echo "[TEST]" > ${job_test_result_file} +echo "comment_description = (no tests yet)" >> ${job_test_result_file} +echo "status = SUCCESS" >> ${job_test_result_file} + +exit 0 diff --git a/bot/test.sh b/bot/test.sh new file mode 100755 index 0000000000..9d978cdcd0 --- /dev/null +++ b/bot/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# Dummy script, no tests yet +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# + +exit 0