Merge pull request #1593 from boegel/easybuild_namespace

easybuild is a namespace package (2nd attempt) (REVIEW)

easybuild/__init__.py

@@ -26,8 +26,7 @@
 Declares EasyBuild namespace, in an extendable way.
 
 @author: Jens Timmerman (Ghent University)
+@author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/toolchains/__init__.py

@@ -28,7 +28,5 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/toolchains/compiler/__init__.py

@@ -28,8 +28,5 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/toolchains/fft/__init__.py

@@ -28,8 +28,5 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/toolchains/linalg/__init__.py

@@ -28,8 +28,5 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/toolchains/mpi/__init__.py

@@ -28,7 +28,5 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/__init__.py

@@ -32,7 +32,5 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/deprecated/__init__.py

@@ -32,7 +32,5 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/include.py

@@ -30,6 +30,7 @@
 """
 import os
 import sys
+from pkg_resources import fixup_namespace_packages
 from vsc.utils import fancylogger
 
 from easybuild.tools.build_log import EasyBuildError
@@ -56,27 +57,26 @@
 # body for __init__.py file in package directory, which takes care of making sure the package can be distributed
 # across multiple directories
 PKG_INIT_BODY = """
-from pkgutil import extend_path
-
-# extend path so Python knows this is not the only place to look for modules in this package
-__path__ = extend_path(__path__, __name__)
+import pkg_resources
+pkg_resources.declare_namespace(__name__)
 """
 
 # more extensive __init__.py specific to easybuild.easyblocks package;
 # this is required because of the way in which the easyblock Python modules are organised in the easybuild-easyblocks
 # repository, i.e. in first-letter subdirectories
 EASYBLOCKS_PKG_INIT_BODY = """
-from pkgutil import extend_path
+import pkg_resources
+import pkgutil
 
 # extend path so Python finds our easyblocks in the subdirectories where they are located
 subdirs = [chr(l) for l in range(ord('a'), ord('z') + 1)] + ['0']
 for subdir in subdirs:
-    __path__ = extend_path(__path__, '%s.%s' % (__name__, subdir))
+    __path__ = pkgutil.extend_path(__path__, '%s.%s' % (__name__, subdir))
 
 # extend path so Python knows this is not the only place to look for modules in this package
-__path__ = extend_path(__path__, __name__)
+pkg_resources.declare_namespace(__name__)
 
-del subdir, subdirs, l
+del l, subdir, subdirs
 """
 
 
@@ -88,7 +88,7 @@ def create_pkg(path, pkg_init_body=None):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        # put __init__.py files in place, with required pkgutil.extend_path statement
+        # put __init__.py files in place, with required pkg_resources.declare_namespace statement
         # note: can't use write_file, since that required build options to be initialised
         with open(init_path, 'w') as handle:
             if pkg_init_body is None:
@@ -170,12 +170,23 @@ def include_easyblocks(tmpdir, paths):
     _log.debug("Included generic easyblocks: %s", included_generic_ebs)
     _log.debug("Included software-specific easyblocks: %s", included_ebs)
 
-    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
+    # prepend new location to Python search path
     sys.path.insert(0, easyblocks_path)
-    reload(easybuild)
-    if 'easybuild.easyblocks' in sys.modules:
-        reload(easybuild.easyblocks)
-        reload(easybuild.easyblocks.generic)
+    fixup_namespace_packages(easyblocks_path)
+
+    # make sure easybuild.easyblocks(.generic) are imported;
+    # reload them to pick up all possible paths they may be in
+    import easybuild.easyblocks
+    reload(easybuild.easyblocks)
+    import easybuild.easyblocks.generic
+    reload(easybuild.easyblocks.generic)
+
+    # hard inject location to included (generic) easyblocks into Python search path
+    # only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
+    new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')
+    easybuild.easyblocks.__path__.insert(0, new_path)
+    new_path = os.path.join(new_path, 'generic')
+    easybuild.easyblocks.generic.__path__.insert(0, new_path)
 
     # sanity check: verify that included easyblocks can be imported (from expected location)
     for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
@@ -205,8 +216,14 @@ def include_module_naming_schemes(tmpdir, paths):
 
     # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
     sys.path.insert(0, mns_path)
+    fixup_namespace_packages(mns_path)
     reload(easybuild.tools.module_naming_scheme)
 
+    # hard inject location to included module naming schemes into Python search path
+    # only prepending to sys.path is not enough due to 'declare_namespace' in module_naming_scheme/__init__.py
+    new_path = os.path.join(mns_path, 'easybuild', 'tools', 'module_naming_scheme')
+    easybuild.tools.module_naming_scheme.__path__.insert(0, new_path)
+
     # sanity check: verify that included module naming schemes can be imported (from expected location)
     verify_imports([os.path.splitext(mns)[0] for mns in included_mns], 'easybuild.tools.module_naming_scheme', mns_dir)
 
@@ -246,9 +263,16 @@ def include_toolchains(tmpdir, paths):
 
     # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
     sys.path.insert(0, toolchains_path)
+    fixup_namespace_packages(toolchains_path)
+
+    # reload toolchain modules and hard inject location to included toolchains into Python search path
+    # only prepending to sys.path is not enough due to 'declare_namespace' in toolchains/*/__init__.py
     reload(easybuild.toolchains)
+    easybuild.toolchains.__path__.insert(0, os.path.join(toolchains_path, 'easybuild', 'toolchains'))
     for subpkg in toolchain_subpkgs:
-        reload(sys.modules['easybuild.toolchains.%s' % subpkg])
+        tcpkg = 'easybuild.toolchains.%s' % subpkg
+        reload(sys.modules[tcpkg])
+        sys.modules[tcpkg].__path__.insert(0, os.path.join(toolchains_path, 'easybuild', 'toolchains', subpkg))
 
     # sanity check: verify that included toolchain modules can be imported (from expected location)
     verify_imports([os.path.splitext(mns)[0] for mns in included_toolchains], 'easybuild.toolchains', tcs_dir)

easybuild/tools/job/__init__.py

@@ -28,7 +28,5 @@
 @author: Jens Timmerman (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/module_naming_scheme/__init__.py

@@ -28,13 +28,12 @@
 @author: Jens Timmerman (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
+import pkg_resources
 
 # required for backward compatibility
 from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme
 
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+pkg_resources.declare_namespace(__name__)
 
 # suffix for devel module filename
 DEVEL_MODULE_SUFFIX = '-easybuild-devel'

easybuild/tools/package/__init__.py

@@ -32,7 +32,5 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/package/package_naming_scheme/__init__.py

@@ -31,7 +31,5 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/repository/__init__.py

@@ -28,7 +28,5 @@
 @author: Jens Timmerman (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

easybuild/tools/toolchain/__init__.py

@@ -50,10 +50,8 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)
 
 # name/version for dummy toolchain
 # if name==DUMMY_TOOLCHAIN_NAME and version==DUMMY_TOOLCHAIN_VERSION, do not load dependencies

setup.py

@@ -110,8 +110,12 @@ def find_rel_test():
     provides=["eb"] + easybuild_packages,
     test_suite="test.framework.suite",
     zip_safe=False,
-    install_requires=["vsc-base >= 2.2.6"],
+    install_requires=[
+        'setuptools >= 0.6',
+        "vsc-base >= 2.4.18",
+    ],
     extras_require = {
         'yeb': ["PyYAML >= 3.11"],
     },
+    namespace_packages=['easybuild'],
 )

test/__init__.py

@@ -32,7 +32,5 @@
 @author: Jens Timmerman (Ghent University)
 @author: Toon Willems (Ghent University)
 """
-from pkgutil import extend_path
-
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

test/framework/build_log.py

@@ -49,6 +49,7 @@ class BuildLogTest(EnhancedTestCase):
 
     def tearDown(self):
         """Cleanup after test."""
+        super(BuildLogTest, self).tearDown()
         # restore default logging format
         setLogFormat(LOGGING_FORMAT)
 

test/framework/easyblock.py

@@ -33,6 +33,7 @@
 import shutil
 import sys
 import tempfile
+from pkg_resources import fixup_namespace_packages
 from test.framework.utilities import EnhancedTestCase, init_config
 from unittest import TestLoader, main
 
@@ -633,6 +634,7 @@ def test_get_easyblock_instance(self):
         eb_blocks_path = os.path.join(testdir, 'sandbox')
         if eb_blocks_path not in sys.path:
             sys.path.append(eb_blocks_path)
+            fixup_namespace_packages(eb_blocks_path)
             easybuild = reload(easybuild)
 
         import easybuild.easyblocks

test/framework/module_generator.py

@@ -30,15 +30,10 @@
 """
 
 import os
-import shutil
-import sys
 import tempfile
-from test.framework.utilities import EnhancedTestCase, init_config
-from unittest import TestLoader, TestSuite, TextTestRunner, main
+from unittest import TestLoader, TestSuite, TextTestRunner
 from vsc.utils.fancylogger import setLogLevelDebug, logToScreen
-from vsc.utils.missing import get_subclasses
 
-import easybuild.tools.module_generator
 from easybuild.framework.easyconfig.tools import process_easyconfig
 from easybuild.tools import config
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl
@@ -47,7 +42,7 @@
 from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.utilities import quote_str
-from test.framework.utilities import find_full_path, init_config
+from test.framework.utilities import EnhancedTestCase, find_full_path, init_config
 
 
 class ModuleGeneratorTest(EnhancedTestCase):
@@ -493,13 +488,6 @@ def test_mns():
         }
         self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', ActiveMNS().det_full_module_name(non_parsed))
 
-        # install custom module naming scheme dynamically
-        test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
-        sys.path.append(test_mns_parent_dir)
-        reload(easybuild)
-        reload(easybuild.tools)
-        reload(easybuild.tools.module_naming_scheme)
-
         # make sure test module naming schemes are available
         mns_mods = ['broken_module_naming_scheme', 'test_module_naming_scheme', 'test_module_naming_scheme_more']
         for test_mns_mod in mns_mods:

test/framework/options.py

@@ -33,7 +33,6 @@
 import shutil
 import sys
 import tempfile
-from test.framework.utilities import EnhancedTestCase, init_config
 from unittest import TestLoader
 from unittest import main as unittestmain
 from urllib2 import URLError
@@ -56,6 +55,7 @@
 from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX
 from easybuild.tools.run import run_cmd
 from easybuild.tools.version import VERSION
+from test.framework.utilities import EnhancedTestCase, init_config
 from vsc.utils import fancylogger
 
 
@@ -1787,6 +1787,11 @@ def test_include_easyblocks(self):
 
         # 'undo' import of foo easyblock
         del sys.modules['easybuild.easyblocks.foo']
+        sys.path = self.orig_sys_path
+        import easybuild.easyblocks
+        reload(easybuild.easyblocks)
+        import easybuild.easyblocks.generic
+        reload(easybuild.easyblocks.generic)
 
         # include extra test easyblocks
         foo_txt = '\n'.join([
@@ -1859,6 +1864,10 @@ def test_include_generic_easyblocks(self):
         # 'undo' import of foobar easyblock
         del sys.modules['easybuild.easyblocks.generic.foobar']
         os.remove(os.path.join(self.test_prefix, 'generic', 'foobar.py'))
+        sys.path = self.orig_sys_path
+        import easybuild.easyblocks
+        reload(easybuild.easyblocks)
+        import easybuild.easyblocks.generic
         reload(easybuild.easyblocks.generic)
 
         error_msg = "Failed to obtain class for FooBar easyblock"