Merge pull request #21 from ORNL/JoshuaSBrown-module-upload-only-what…

…-is-needed

Attempt to reduce cost of module imports, by only importing files tha…

.github/workflows/build.yml

@@ -47,4 +47,5 @@ jobs:
       - name: Build documentation
         run: |
           cd docs
+          pip install -r requirements.txt
           make html

tests/test_plugins.py

@@ -17,14 +17,18 @@ def test_registered_plugins():
     plugins = Plugins()
     found_shell = False
     found_rsync = False
+    found_globus = False
     for plugin in plugins.registered:
         if plugin == "shell":
             found_shell = True
         elif plugin == "rsync":
             found_rsync = True
+        elif plugin == "globus":
+            found_globus = True
 
     assert found_shell
     assert found_rsync
+    assert found_globus
 
 
 @pytest.mark.unit

zambeze/orchestration/plugins.py

@@ -41,33 +41,17 @@ def __init__(self, logger: Optional[logging.Logger] = None) -> None:
         self.__logger: logging.Logger = (
             logging.getLogger(__name__) if logger is None else logger
         )
+        self.__module_names = []
+        self._plugins = {}
         self.__registerPlugins()
 
     def __registerPlugins(self) -> None:
         """Will register all the plugins provided in the plugin_modules folder"""
-        self._plugins = {}
-        module_names = []
-
         plugin_path = [str(Path(__file__).resolve().parent) + "/plugin_modules"]
         for importer, module_name, ispkg in pkgutil.walk_packages(path=plugin_path):
-            if module_name == "abstract_plugin":
-                continue
-
-            module = import_module(
-                f"zambeze.orchestration.plugin_modules.{module_name}"
-            )
-            module_names.append(module_name)
-            for attribute_name in dir(module):
-                potential_plugin = getattr(module, attribute_name)
-                if isclass(potential_plugin):
-                    if (
-                        issubclass(potential_plugin, Plugin)
-                        and attribute_name != "Plugin"
-                    ):
-                        self._plugins[attribute_name.lower()] = potential_plugin(
-                            logger=self.__logger
-                        )
-        self.__logger.debug(f"Registered Plugins: {', '.join(module_names)}")
+            if module_name != "abstract_plugin" and module_name != "__init__":
+                self.__module_names.append(module_name)
+        self.__logger.debug(f"Registered Plugins: {', '.join(self.__module_names)}")
 
     @property
     def registered(self) -> list[Plugin]:
@@ -90,9 +74,9 @@ def registered(self) -> list[Plugin]:
         >>> shell
         >>> rsync
         """
-        plugins: list[Plugin] = []
-        for key in self._plugins:
-            plugins.append(deepcopy(key))
+        plugins: list[str] = []
+        for module_name in self.__module_names:
+            plugins.append(deepcopy(module_name))
         return plugins
 
     def configure(self, config: dict):
@@ -129,6 +113,21 @@ def configure(self, config: dict):
 
         This will just configure the "shell" plugin
         """
+        for module_name in self.__module_names:
+            module = import_module(
+                f"zambeze.orchestration.plugin_modules.{module_name}"
+            )
+            for attribute_name in dir(module):
+                potential_plugin = getattr(module, attribute_name)
+                if isclass(potential_plugin):
+                    if (
+                        issubclass(potential_plugin, Plugin)
+                        and attribute_name != "Plugin"
+                    ):
+                        self._plugins[attribute_name.lower()] = potential_plugin(
+                            logger=self.__logger
+                        )
+
         for key in self._plugins:
             if key in config.keys():
                 obj = self._plugins.get(key)