Merge pull request #4107 from boegel/fix_easyconfigs_cache

also use EasyConfig instances cache in process_easyconfig when build_specs is empty dict

easybuild/framework/easyconfig/easyconfig.py

@@ -2054,7 +2054,7 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
 
     # only cache when no build specifications are involved (since those can't be part of a dict key)
     cache_key = None
-    if build_specs is None:
+    if not build_specs:
         cache_key = (path, validate, hidden, parse_only)
         if cache_key in _easyconfigs_cache:
             return [e.copy() for e in _easyconfigs_cache[cache_key]]

test/framework/easyconfig.py

@@ -4723,6 +4723,56 @@ def test_ARCH(self):
         arch = easyconfig.constants.EASYCONFIG_CONSTANTS['ARCH'][0]
         self.assertTrue(arch in KNOWN_ARCH_CONSTANTS, "Unexpected value for ARCH constant: %s" % arch)
 
+    def test_easyconfigs_caches(self):
+        """
+        Test whether easyconfigs caches work as intended.
+        """
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        libtoy_ec = os.path.join(test_ecs_dir, 'l', 'libtoy', 'libtoy-0.0.eb')
+        toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')
+        copy_file(libtoy_ec, self.test_prefix)
+        copy_file(toy_ec, self.test_prefix)
+        libtoy_ec = os.path.join(self.test_prefix, os.path.basename(libtoy_ec))
+        toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
+
+        ec1 = process_easyconfig(toy_ec)[0]
+        self.assertEqual(ec1['ec'].name, 'toy')
+        self.assertEqual(ec1['ec'].version, '0.0')
+        self.assertTrue(isinstance(ec1['ec'].toolchain, SystemToolchain))
+        self.assertTrue(os.path.samefile(ec1['ec'].path, toy_ec))
+
+        # wipe toy easyconfig (but path still needs to exist)
+        write_file(toy_ec, '')
+
+        # check if cached EasyConfig instance is picked up when calling process_easyconfig again
+        ec2 = process_easyconfig(toy_ec)[0]
+        self.assertEqual(ec2['ec'].name, 'toy')
+        self.assertEqual(ec2['ec'].version, '0.0')
+        self.assertTrue(isinstance(ec2['ec'].toolchain, SystemToolchain))
+        self.assertTrue(os.path.samefile(ec2['ec'].path, toy_ec))
+
+        # also check whether easyconfigs cache works with end-to-end test
+        args = [libtoy_ec, '--trace']
+        self.mock_stdout(True)
+        self.eb_main(args, do_build=True, testing=False, raise_error=True, clear_caches=False)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        regex = re.compile(r"generating module file @ .*/modules/all/libtoy/0.0", re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        # wipe libtoy easyconfig (but path still needs to exist)
+        write_file(libtoy_ec, '')
+
+        # retrying installation of libtoy easyconfig should not fail, thanks to easyconfigs cache
+        self.mock_stdout(True)
+        self.eb_main(args, do_build=True, testing=False, raise_error=True, clear_caches=False)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        regex = re.compile(r"libtoy/0\.0 is already installed", re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
 
 def suite():
     """ returns all the testcases in this module """

test/framework/utilities.py

@@ -285,9 +285,10 @@ def reset_modulepath(self, modpaths):
         self.modtool.set_mod_paths()
 
     def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False,
-                reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True):
+                reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True, clear_caches=True):
         """Helper method to call EasyBuild main function."""
-        cleanup()
+
+        cleanup(clear_caches=clear_caches)
 
         # always run main in unit testing mode (which for example allows for using deprecated toolchains);
         # note: don't change 'args' value, which is passed by reference!
@@ -328,7 +329,7 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos
 
         if redo_init_config:
             # make sure config is reinitialized
-            init_config(with_include=False)
+            init_config(with_include=False, clear_caches=clear_caches)
 
         # restore environment to what it was before running main,
         # changes may have been made by eb_main (e.g. $TMPDIR & co)
@@ -445,26 +446,28 @@ def loadTestsFromTestCase(self, test_case_class, filters):
         return self.suiteClass(test_cases)
 
 
-def cleanup():
+def cleanup(clear_caches=True):
     """Perform cleanup of singletons and caches."""
+
     # clear Singleton instances, to start afresh
     Singleton._instances.clear()
 
-    # empty caches
-    tc_utils._initial_toolchain_instances.clear()
-    easyconfig._easyconfigs_cache.clear()
-    easyconfig._easyconfig_files_cache.clear()
-    easyconfig.get_toolchain_hierarchy.clear()
-    mns_toolchain._toolchain_details_cache.clear()
+    # clear various caches, to start with a clean slate
+    if clear_caches:
+        tc_utils._initial_toolchain_instances.clear()
+        easyconfig._easyconfigs_cache.clear()
+        easyconfig._easyconfig_files_cache.clear()
+        easyconfig.get_toolchain_hierarchy.clear()
+        mns_toolchain._toolchain_details_cache.clear()
 
     # reset to make sure tempfile picks up new temporary directory to use
     tempfile.tempdir = None
 
 
-def init_config(args=None, build_options=None, with_include=True):
+def init_config(args=None, build_options=None, with_include=True, clear_caches=True):
     """(re)initialize configuration"""
 
-    cleanup()
+    cleanup(clear_caches=clear_caches)
 
     # initialize configuration so config.get_modules_tool function works
     eb_go = eboptions.parse_options(args=args, with_include=with_include)