Merge pull request #1 from lexming/PR4248

Update for PR 4248
easybuilders · Mar 1, 2024 · e733cf4 · e733cf4
2 parents 4f8ddb8 + 8c76561
commit e733cf4
Show file tree

Hide file tree

Showing 99 changed files with 3,847 additions and 1,217 deletions.
diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
@@ -74,7 +74,7 @@ jobs:
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
-          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test
       run: |
@@ -95,7 +95,7 @@ jobs:
           echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
           # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
           # see https://docs.easybuild.io/en/latest/Containers.html
-          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
           export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image

diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
@@ -66,7 +66,7 @@ jobs:
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
-          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test
       run: |
@@ -87,7 +87,7 @@ jobs:
           echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
           # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
           # see https://docs.easybuild.io/en/latest/Containers.html
-          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
           export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
           export EASYBUILD_CONTAINER_TYPE='apptainer'

diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -0,0 +1,60 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/5.0.x.tar.gz
+              tar xfz 5.0.x.tar.gz
+              rm -f 5.0.x.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-5.0.x:$HOME/easybuild-easyconfigs-5.0.x" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -13,10 +13,7 @@ jobs:
   setup:
     runs-on: ubuntu-20.04
     outputs:
-        lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
-        modulesTcl: modules-tcl-1.147
-        modules3: modules-3.2.10
         modules4: modules-4.1.4
     steps:
       - run: "true"
@@ -29,49 +26,25 @@ jobs:
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
-          - ${{needs.setup.outputs.lmod7}}
           - ${{needs.setup.outputs.lmod8}}
-          - ${{needs.setup.outputs.modulesTcl}}
-          - ${{needs.setup.outputs.modules3}}
           - ${{needs.setup.outputs.modules4}}
-        module_syntax: [Lua, Tcl]
         lc_all: [""]
-        # don't test with Lua module syntax (only supported in Lmod)
-        exclude:
-          - modules_tool: ${{needs.setup.outputs.modulesTcl}}
-            module_syntax: Lua
-          - modules_tool: ${{needs.setup.outputs.modules3}}
-            module_syntax: Lua
-          - modules_tool: ${{needs.setup.outputs.modules4}}
-            module_syntax: Lua
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
           - python: 3.7
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: 3.8
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
-          - python: 3.8
-            modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Tcl
           - python: 3.9
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: '3.10'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
-          - python: '3.11'
-            modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
           - python: '3.11'
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Tcl
           # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
           # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
             modules_tool: ${{needs.setup.outputs.lmod8}}
-            module_syntax: Lua
             lc_all: C
       fail-fast: false
     steps:
@@ -122,16 +95,11 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
-        # and only when testing with Lua as module syntax,
-        # to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
+        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            if [ "x${{matrix.python}}" == 'x2.6' ];
-              then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
-              else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            fi;
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
             python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
           fi
           echo "GitHub token installed!"
@@ -169,8 +137,6 @@ jobs:
     - name: run test suite
       env:
         EB_VERBOSE: 1
-        EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
-        TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
         LC_ALL: ${{matrix.lc_all}}
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
@@ -186,28 +152,37 @@ jobs:
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
           fi
-          export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL
-          eb --show-config
-          # gather some useful info on test system
-          eb --show-system-info
-          # check GitHub configuration
-          eb --check-github --github-user=easybuild_test
-          # create file owned by root but writable by anyone (used by test_copy_file)
-          sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          # run test suite
-          python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
-          # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
-          # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
-          PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
-          test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
+          export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL}
+
+          # Run tests with LUA and Tcl module syntax (where supported)
+          for module_syntax in Lua Tcl; do
+            # Only Lmod supports Lua
+            if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then
+              echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
+              continue
+            fi
+            printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
+            export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
+            export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
+
+            eb --show-config
+            # gather some useful info on test system
+            eb --show-system-info
+            # check GitHub configuration
+            eb --check-github --github-user=easybuild_test
+            # create file owned by root but writable by anyone (used by test_copy_file)
+            sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+            sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+            # run test suite
+            python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
+            # try and make sure output of running tests is clean (no printed messages/warnings)
+            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
+            PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
+            test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
+          done
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -4,6 +4,81 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.0 (30 December 2023)
+-------------------------
+
+feature release
+
+- various enhancements, including:
+  - allow tweaking of easyconfigs from different toolchains (#3669)
+  - add support for `--list-software --output-format=json` (#4152)
+  - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403)
+- various bug fixes, including:
+  - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361)
+  - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369)
+  - adapt module function check for Environment Modules v4+ (#4371)
+  - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375)
+  - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377)
+  - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379)
+  - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385)
+  - eliminate duplicate multideps from generated module files (#4386)
+  - resolve templated values in extension names in `_make_extension_list` (#4392)
+  - use source toolchain version when passing only `--try-toolchain` (#4395)
+  - fix writing spider cache for Lmod >= 8.7.12 (#4402)
+  - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405)
+  - fix `LooseVersion` when running with Python 2.7 (#4408)
+  - use more recent easyblocks PR in `test_github_merge_pr` (#4414)
+- other changes:
+  - extend test that checks build environment to recent `foss/2023a` toolchain (#4391)
+
+
+v4.8.2 (29 October 2023)
+------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for `%(sysroot)s` template value (#4359)
+  - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360)
+- various bug fixes, including:
+  - add CI workflow to run unit tests with Python 2 (again) (#4333)
+  - fix typo in help message for `--silence-hook-trigger` (#4343)
+  - include major version (`*majver`) templates in auto-generated documentation (#4347)
+  - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350)
+  - fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365)
+  - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366)
+- other changes:
+  - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192)
+
+
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+  - suggest default title in `--review-pr` (#4287)
+  - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+  - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+  - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+  - add `run_shell_cmd` hook (#4323)
+  - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+  - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+  - add support for using fine grained Github tokens (#4332)
+  - add definitions for ifbf and iofbf toolchain (#4337)
+  - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+  - improve error when checksum dict has no entry for a file (#4150)
+  - avoid error being logged when `checksums.json` is not found (#4261)
+  - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+  - ignore request for external module (meta)data when no modules tool is active (#4308)
+  - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+  - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+  - enhance `Toolchain.get_flag` to handle lists (#4319)
+  - only add extensions in module file if there are extensions (#4331)
+
+
 v4.8.0 (7 July 2023)
 --------------------
 

diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py
@@ -285,7 +285,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non
         overwrite make record to use a fancy record (with more options)
         """
         logrecordcls = logging.LogRecord
-        if hasattr(self, 'fancyrecord') and self.fancyrecord:
+        if getattr(self, 'fancyrecord', None):
             logrecordcls = FancyLogRecord
         try:
             new_msg = str(msg)