Merge pull request #3311 from migueldiascosta/20200430163846_new_pr_c…

…worcWSdYy

allow including easyblocks from multiple locations

easybuild/tools/include.py

@@ -125,6 +125,8 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None
 
 def verify_imports(pymods, pypkg, from_path):
     """Verify that import of specified modules from specified package and expected location works."""
+    saved_modules = sys.modules.copy()
+
     for pymod in pymods:
         pymod_spec = '%s.%s' % (pypkg, pymod)
         try:
@@ -140,6 +142,10 @@ def verify_imports(pymods, pypkg, from_path):
 
         _log.debug("Import of %s from %s verified", pymod_spec, from_path)
 
+    # restore sys.modules to its original state (not only verified modules but also their dependencies)
+    sys.modules.clear()
+    sys.modules.update(saved_modules)
+
 
 def is_software_specific_easyblock(module):
     """Determine whether Python module at specified location is a software-specific easyblock."""

easybuild/tools/options.py

@@ -58,7 +58,7 @@
 from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM
 from easybuild.tools import build_log, run  # build_log should always stay there, to ensure EasyBuildLog
 from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
-from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror
+from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror
 from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES
 from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE
 from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
@@ -76,7 +76,8 @@
 from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
 from easybuild.tools.docs import list_easyblocks, list_toolchains
 from easybuild.tools.environment import restore_env, unset_env_vars
-from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, install_fake_vsc, move_file, which
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc
+from easybuild.tools.filetools import move_file, which
 from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED
 from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS
 from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
@@ -1451,17 +1452,23 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False):
     init_build_options(build_options=build_options, cmdline_options=options)
 
     # done here instead of in _postprocess_include because github integration requires build_options to be initialized
-    if eb_go.options.include_easyblocks_from_pr:
-        easyblocks_from_pr = fetch_easyblocks_from_pr(eb_go.options.include_easyblocks_from_pr)
+    pr_easyblocks = eb_go.options.include_easyblocks_from_pr
+    if pr_easyblocks:
+        easyblocks_from_pr = fetch_easyblocks_from_pr(pr_easyblocks)
+        included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr])
 
         if eb_go.options.include_easyblocks:
-            # make sure we're not including the same easyblock twice
-            included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr])
-            included_from_file = set([os.path.basename(eb) for eb in eb_go.options.include_easyblocks])
+            # check if you are including the same easyblock twice
+            included_paths = expand_glob_paths(eb_go.options.include_easyblocks)
+            included_from_file = set([os.path.basename(eb) for eb in included_paths])
             included_twice = included_from_pr & included_from_file
             if included_twice:
-                raise EasyBuildError("Multiple inclusion of %s, check your --include-easyblocks options",
-                                     ','.join(included_twice))
+                warning_msg = "One or more easyblocks included from multiple locations: %s " % ', '.join(included_twice)
+                warning_msg += "(the one(s) from PR #%s will be used)" % pr_easyblocks
+                print_warning(warning_msg)
+
+        for easyblock in included_from_pr:
+            print_msg("easyblock %s included from PR #%s" % (easyblock, pr_easyblocks), log=log)
 
         include_easyblocks(eb_go.options.tmpdir, easyblocks_from_pr)
 

test/framework/include.py

@@ -102,19 +102,22 @@ def test_include_easyblocks(self):
         myfoo_pyc_path = easybuild.easyblocks.myfoo.__file__
         myfoo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(myfoo_pyc_path), 'myfoo.py'))
         self.assertTrue(os.path.samefile(up(myfoo_real_py_path, 1), myeasyblocks))
+        del sys.modules['easybuild.easyblocks.myfoo']
 
         import easybuild.easyblocks.generic.mybar
         mybar_pyc_path = easybuild.easyblocks.generic.mybar.__file__
         mybar_real_py_path = os.path.realpath(os.path.join(os.path.dirname(mybar_pyc_path), 'mybar.py'))
         self.assertTrue(os.path.samefile(up(mybar_real_py_path, 2), myeasyblocks))
+        del sys.modules['easybuild.easyblocks.generic.mybar']
 
         # existing (test) easyblocks are unaffected
         import easybuild.easyblocks.foofoo
         foofoo_path = os.path.dirname(os.path.dirname(easybuild.easyblocks.foofoo.__file__))
         self.assertTrue(os.path.samefile(foofoo_path, test_easyblocks))
+        del sys.modules['easybuild.easyblocks.foofoo']
 
     def test_include_easyblocks_priority(self):
-        """Test whether easyblocks included via include_easyblocks() get prioroity over others."""
+        """Test whether easyblocks included via include_easyblocks() get priority over others."""
         test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks')
 
         # make sure that test 'foo' easyblock is there
@@ -138,15 +141,23 @@ def test_include_easyblocks_priority(self):
             "   pass",
         ])
         write_file(os.path.join(myeasyblocks, 'foo.py'), foo_easyblock_txt)
+
+        # check that the sandboxed easyblock is imported before include_easyblocks is run
+        foo_pyc_path = easybuild.easyblocks.foo.__file__
+        foo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(foo_pyc_path), 'foo.py'))
+        self.assertTrue(os.path.samefile(os.path.dirname(os.path.dirname(foo_pyc_path)), test_easyblocks))
+        self.assertFalse(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py')))
+
         include_easyblocks(self.test_prefix, [os.path.join(myeasyblocks, 'foo.py')])
 
+        # check that the included easyblock is imported after include_easyblocks is run
         foo_pyc_path = easybuild.easyblocks.foo.__file__
         foo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(foo_pyc_path), 'foo.py'))
-        self.assertFalse(os.path.samefile(os.path.dirname(foo_pyc_path), test_easyblocks))
+        self.assertFalse(os.path.samefile(os.path.dirname(os.path.dirname(foo_pyc_path)), test_easyblocks))
         self.assertTrue(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py')))
 
-        # 'undo' import of foo easyblock
-        del sys.modules['easybuild.easyblocks.foo']
+        # check that the included easyblock is not loaded
+        self.assertFalse('easybuild.easyblocks.foo' in sys.modules)
 
     def test_include_mns(self):
         """Test include_module_naming_schemes()."""

test/framework/options.py

@@ -2885,9 +2885,17 @@ def test_xxx_include_easyblocks_from_pr(self):
             '--unittest-file=%s' % self.logfile,
             '--github-user=%s' % GITHUB_TEST_ACCOUNT,
         ]
+        self.mock_stderr(True)
+        self.mock_stdout(True)
         self.eb_main(args, logfile=dummylogfn, raise_error=True)
+        stderr, stdout = self.get_stderr(), self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
         logtxt = read_file(self.logfile)
 
+        self.assertFalse(stderr)
+        self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #1915\n")
+
         # easyblock included from pr is found
         path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks')
         cmm_pattern = os.path.join(path_pattern, 'generic', 'cmakemake.py')
@@ -2917,19 +2925,46 @@ def test_xxx_include_easyblocks_from_pr(self):
         ])
         write_file(os.path.join(self.test_prefix, 'cmakemake.py'), cmm_txt)
 
-        # including the same easyblock twice should fail
+        # including the same easyblock twice should work and give priority to the one from the PR
         args = [
-            '--include-easyblocks=%s/cmakemake.py' % self.test_prefix,
+            '--include-easyblocks=%s/*.py' % self.test_prefix,
             '--include-easyblocks-from-pr=1915',
             '--list-easyblocks=detailed',
             '--unittest-file=%s' % self.logfile,
             '--github-user=%s' % GITHUB_TEST_ACCOUNT,
         ]
-        self.assertErrorRegex(EasyBuildError,
-                              "Multiple inclusion of cmakemake.py, check your --include-easyblocks options",
-                              self.eb_main, args, raise_error=True)
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.eb_main(args, logfile=dummylogfn, raise_error=True)
+        stderr, stdout = self.get_stderr(), self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        logtxt = read_file(self.logfile)
+
+        expected = "WARNING: One or more easyblocks included from multiple locations: "
+        expected += "cmakemake.py (the one(s) from PR #1915 will be used)"
+        self.assertEqual(stderr.strip(), expected)
+        self.assertEqual(stdout, "== easyblock cmakemake.py included from PR #1915\n")
 
+        # easyblock included from pr is found
+        path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks')
+        cmm_pattern = os.path.join(path_pattern, 'generic', 'cmakemake.py')
+        cmm_regex = re.compile(r"\|-- CMakeMake \(easybuild.easyblocks.generic.cmakemake @ %s\)" % cmm_pattern, re.M)
+        self.assertTrue(cmm_regex.search(logtxt), "Pattern '%s' found in: %s" % (cmm_regex.pattern, logtxt))
+
+        # easyblock is found via get_easyblock_class
+        klass = get_easyblock_class('CMakeMake')
+        self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass)
+
+        # 'undo' import of easyblocks
+        del sys.modules['easybuild.easyblocks.foo']
+        del sys.modules['easybuild.easyblocks.generic.cmakemake']
         os.remove(os.path.join(self.test_prefix, 'cmakemake.py'))
+        sys.path = orig_local_sys_path
+        import easybuild.easyblocks
+        reload(easybuild.easyblocks)
+        import easybuild.easyblocks.generic
+        reload(easybuild.easyblocks.generic)
 
         # clear log
         write_file(self.logfile, '')
@@ -2941,9 +2976,17 @@ def test_xxx_include_easyblocks_from_pr(self):
             '--github-user=%s' % GITHUB_TEST_ACCOUNT,
             '--extended-dry-run',
         ]
+        self.mock_stderr(True)
+        self.mock_stdout(True)
         self.eb_main(args, logfile=dummylogfn, raise_error=True)
+        stderr, stdout = self.get_stderr(), self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
         logtxt = read_file(self.logfile)
 
+        self.assertFalse(stderr)
+        self.assertEqual(stdout, "== easyblock cmake.py included from PR #1936\n")
+
         # easyconfig from pr is found
         ec_pattern = os.path.join(self.test_prefix, '.*', 'files_pr10487', 'c', 'CMake',
                                   'CMake-3.16.4-GCCcore-9.3.0.eb')