Merge pull request #3906 from boegel/required_deps_None

fall back to sequential installation for extensions with unknown dependencies

easybuild/framework/easyblock.py

@@ -1852,7 +1852,26 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                 # check whether extension at top of the queue is ready to install
                 ext = exts_queue.pop(0)
 
-                pending_deps = [x for x in ext.required_deps if x not in installed_ext_names]
+                required_deps = ext.required_deps
+                if required_deps is None:
+                    pending_deps = None
+                    self.log.info("Required dependencies for %s are unknown!", ext.name)
+                else:
+                    self.log.info("Required dependencies for %s: %s", ext.name, ', '.join(required_deps))
+                    pending_deps = [x for x in required_deps if x not in installed_ext_names]
+                    self.log.info("Missing required dependencies for %s: %s", ext.name, ', '.join(pending_deps))
+
+                # if required dependencies could not be determined, wait until all preceding extensions are installed
+                if pending_deps is None:
+                    if running_exts:
+                        # add extension back at top of the queue,
+                        # since we need to preverse installation order of extensions;
+                        # break out of for loop since there is no point to keep checking
+                        # until running installations have been completed
+                        exts_queue.insert(0, ext)
+                        break
+                    else:
+                        pending_deps = []
 
                 if self.dry_run:
                     tup = (ext.name, ext.version, ext.__class__.__name__)
@@ -1865,7 +1884,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
 
                     # make sure all required dependencies are actually going to be installed,
                     # to avoid getting stuck in an infinite loop!
-                    missing_deps = [x for x in ext.required_deps if x not in all_ext_names]
+                    missing_deps = [x for x in required_deps if x not in all_ext_names]
                     if missing_deps:
                         raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s",
                                              ext.name, ', '.join(missing_deps))

easybuild/framework/extension.py

@@ -225,7 +225,8 @@ def async_cmd_check(self):
     @property
     def required_deps(self):
         """Return list of required dependencies for this extension."""
-        raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name)
+        self.log.info("Don't know how to determine required dependencies for extension '%s'", self.name)
+        return None
 
     @property
     def toolchain(self):

test/framework/toy_build.py

@@ -1834,6 +1834,28 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertTrue(regex.search(stdout), error_msg)
 
+        # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method;
+        # framework should fall back to installing extensions sequentially
+        toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py')
+        copy_file(toy_ext_eb, self.test_prefix)
+        toy_ext_eb = os.path.join(self.test_prefix, 'toy_extension.py')
+        toy_ext_eb_txt = read_file(toy_ext_eb)
+        toy_ext_eb_txt = toy_ext_eb_txt.replace('def required_deps', 'def xxx_required_deps')
+        write_file(toy_ext_eb, toy_ext_eb_txt)
+
+        args[-1] = '--include-easyblocks=%s' % toy_ext_eb
+        stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        self.assertEqual(stderr, '')
+        expected_stdout = '\n'.join([
+            "== 0 out of 4 extensions installed (3 queued, 1 running: ls)",
+            "== 1 out of 4 extensions installed (2 queued, 1 running: bar)",
+            "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)",
+            "== 3 out of 4 extensions installed (0 queued, 1 running: toy)",
+            "== 4 out of 4 extensions installed (0 queued, 0 running: )",
+            '',
+        ])
+        self.assertEqual(stdout, expected_stdout)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""