Merge branch 'develop' into 5.0.x

.github/workflows/container_tests.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/container_tests_apptainer.yml

@@ -18,10 +18,10 @@ jobs:
         apptainer: [1.0.0, 1.1.7]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/eb_command.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/linting.yml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/unit_tests.yml

@@ -78,7 +78,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -117,14 +117,16 @@ jobs:
 
     - name: install GitHub token (if available)
       env:
-        # see https://github.com/<username>/easybuild-framework/settings/secrets
-        GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}}
+        # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
+        # this token is not available in pull requests, so tests that require it are skipped in PRs,
+        # and are only run after the PR gets merged
+        GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
         # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
         # and only when testing with Lua as module syntax,
         # to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then
+        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
             if [ "x${{matrix.python}}" == 'x2.6' ];
               then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";

RELEASE_NOTES

@@ -4,6 +4,27 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.7.2 (27 May 2023)
+--------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - extend (experimental) support for generating container images with Apptainer (#3975)
+  - add default download timeout (#4226)
+  - make the RPATH section check with `readelf -d` in sanity check optional  (#4249)
+- various bug fixes, including:
+  - add `make_extension_string` and `_make_extension_list` methods to `EasyBlock` class, so easyblocks can customize them (#3697)
+  - fix typo in condition guarding installation of GitHub token in CI workflow to run unit tests + fix broken tests related to GitHub integration features (#4216)
+  - fix --check-github as gist url now includes username (#4231)
+  - switch to Ubuntu 22.04 in GitHub Actions workflow to run container tests (#4238)
+  - use "/usr/bin/env python" in test_run_cmd_script (#4241)
+- other changes:
+  - print potential errors after running a command only once (#4065)
+  - change github_* template descriptions to be somewhat clearer (#4251)
+  - remove workflow to test bootstrap script (#4255)
+
+
 v4.7.1 (March 20th 2023)
 ------------------------
 

easybuild/framework/easyblock.py

@@ -1427,7 +1427,7 @@ def make_module_extra_extensions(self):
 
         # set environment variable that specifies list of extensions
         # We need only name and version, so don't resolve templates
-        exts_list = ','.join(['-'.join(ext[:2]) for ext in self.cfg.get_ref('exts_list')])
+        exts_list = self.make_extension_string(ext_sep=',', sort=False)
         env_var_name = convert_name(self.name, upper=True)
         lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
 
@@ -1708,6 +1708,27 @@ def load_dependency_modules(self):
     # EXTENSIONS UTILITY FUNCTIONS
     #
 
+    def _make_extension_list(self):
+        """
+        Return a list of extension names and their versions included in this installation
+
+        Each entry should be a (name, version) tuple or just (name, ) if no version exists
+        """
+        # We need only name and version, so don't resolve templates
+        # Each extension in exts_list is either a string or a list/tuple with name, version as first entries
+        return [(ext, ) if isinstance(ext, str) else ext[:2] for ext in self.cfg.get_ref('exts_list')]
+
+    def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
+        """
+        Generate a string with a list of extensions.
+
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        exts_list = (name_version_sep.join(ext) for ext in self._make_extension_list())
+        if sort:
+            exts_list = sorted(exts_list, key=str.lower)
+        return ext_sep.join(exts_list)
+
     def prepare_for_extensions(self):
         """
         Also do this before (eg to set the template)
@@ -3057,7 +3078,7 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs):
         self.cfg['builddependencies'] = builddeps
         self.cfg.iterating = False
 
-    def sanity_check_rpath(self, rpath_dirs=None):
+    def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
         """Sanity check binaries/libraries w.r.t. RPATH linking."""
 
         self.log.info("Checking RPATH linkage for binaries/libraries...")
@@ -3122,17 +3143,21 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             self.log.debug("Output of 'ldd %s' checked, looks OK", path)
 
                         # check whether RPATH section in 'readelf -d' output is there
-                        out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
-                        if ec:
-                            fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
-                        elif not readelf_rpath_regex.search(out):
-                            fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
+                        if check_readelf_rpath:
+                            fail_msg = None
+                            out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
+                            if ec:
+                                fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
+                            elif not readelf_rpath_regex.search(out):
+                                fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out)
+
+                            if fail_msg:
+                                self.log.warning(fail_msg)
+                                fails.append(fail_msg)
+                            else:
+                                self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
                         else:
-                            self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
+                            self.log.debug("Skipping the RPATH section check with 'readelf -d', as requested")
             else:
                 self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
 

easybuild/framework/easyconfig/templates.py

@@ -113,13 +113,15 @@
     ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
      'http download for gnome ftp server'),
     ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive',
-     'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'),
+     'GitHub source URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
     ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
-     'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'),
+     'GitHub source URL with lowercase name (if github_account easyconfig '
+     'parameter is not specified, namelower is used in its place)'),
     ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
-     'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'),
+     'GitHub release URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'),
     ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
-     'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'),
+     'GitHub release URL with lowercase name (if github_account easyconfig '
+     'parameter is not specified, namelower is used in its place)'),
     ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
      'download.savannah.gnu.org source url'),
     ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',

easybuild/tools/module_generator.py

@@ -611,28 +611,17 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None):
 
     def _generate_extension_list(self):
         """
-        Generate a string with a comma-separated list of extensions.
-        """
-        # We need only name and version, so don't resolve templates
-        exts_list = self.app.cfg.get_ref('exts_list')
-        extensions = ', '.join(sorted(['-'.join(ext[:2]) for ext in exts_list], key=str.lower))
+        Generate a string with a list of extensions.
 
-        return extensions
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        return self.app.make_extension_string()
 
     def _generate_extensions_list(self):
         """
         Generate a list of all extensions in name/version format
         """
-        exts_list = self.app.cfg['exts_list']
-        # the format is extension_name/extension_version
-        exts_ver_list = []
-        for ext in exts_list:
-            if isinstance(ext, tuple):
-                exts_ver_list.append('%s/%s' % (ext[0], ext[1]))
-            elif isinstance(ext, str):
-                exts_ver_list.append(ext)
-
-        return sorted(exts_ver_list, key=str.lower)
+        return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',')
 
     def _generate_help_text(self):
         """

easybuild/tools/run.py

@@ -622,19 +622,19 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
     """
     if strictness == IGNORE:
         check_ec = False
-        use_regexp = False
+        fail_on_error_match = False
     elif strictness == WARN:
         check_ec = True
-        use_regexp = False
+        fail_on_error_match = False
     elif strictness == ERROR:
         check_ec = True
-        use_regexp = True
+        fail_on_error_match = True
     else:
         raise EasyBuildError("invalid strictness setting: %s", strictness)
 
     # allow for overriding the regexp setting
     if not regexp:
-        use_regexp = False
+        fail_on_error_match = False
 
     if ec and (log_all or log_ok):
         # We don't want to error if the user doesn't care
@@ -649,14 +649,16 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
             _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
 
     # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
-    if use_regexp or regexp:
-        res = parse_log_for_error(stdouterr, regexp, msg="Command used: %s" % cmd)
-        if len(res) > 0:
-            message = "Found %s errors in command output (output: %s)" % (len(res), "\n\t".join([r[0] for r in res]))
-            if use_regexp:
-                raise EasyBuildError(message)
+    if fail_on_error_match or regexp:
+        res = parse_log_for_error(stdouterr, regexp, stdout=False)
+        if res:
+            errors = "\n\t" + "\n\t".join([r[0] for r in res])
+            error_str = "error" if len(res) == 1 else "errors"
+            if fail_on_error_match:
+                raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
             else:
-                _log.warning(message)
+                _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
+                             len(res), error_str, cmd, errors)
 
     if simple:
         if ec:

test/framework/easyblock.py

@@ -1032,7 +1032,7 @@ def test_init_extensions(self):
         toy_ec_txt = read_file(toy_ec_file)
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
-        test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',")
+        test_ec_txt = toy_ec_txt.replace("('barbar', '1.2', {", "('barbar', '1.2', {'easyblock': 'DummyExtension',")
         write_file(test_ec, test_ec_txt)
         ec = process_easyconfig(test_ec)[0]
         eb = get_easyblock_instance(ec)
@@ -1836,7 +1836,7 @@ def test_collect_exts_file_info(self):
         self.assertIsInstance(exts_file_info, list)
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
@@ -1848,7 +1848,7 @@ def test_collect_exts_file_info(self):
         self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
 
         self.assertEqual(exts_file_info[2]['name'], 'barbar')
-        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz'))
         self.assertNotIn('patches', exts_file_info[2])
 
         self.assertEqual(exts_file_info[3]['name'], 'toy')
@@ -1861,7 +1861,7 @@ def test_collect_exts_file_info(self):
         self.assertIsInstance(exts_file_info, list)
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertNotIn('src', exts_file_info[1])

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb

@@ -27,10 +27,10 @@ exts_default_options = {
 }
 
 exts_list = [
-    'ls',  # extension that is part of "standard library"
+    'ulimit',  # extension that is part of "standard library"
     ('bar', '0.0', {
         'buildopts': " && gcc bar.c -o anotherbar",
-        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for 
+        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for
         # custom extension filter to verify use of stdin value being passed to filter command
         'exts_filter': ("cat | grep '^bar$'", '%(name)s'),
         'patches': [
@@ -43,7 +43,7 @@ exts_list = [
         # cfr. https://github.com/easybuilders/easybuild-framework/pull/3034
         'keepsymlinks': True,
     }),
-    ('barbar', '0.0', {
+    ('barbar', '1.2', {
         'start_dir': 'src',
     }),
     (name, version, {