diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 5e5f7a9444..7231e1aeda 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -52,7 +52,8 @@ def det_lib_path(): def find_egg_dir_for(path, pkg): """Find full path of egg dir for given package.""" - + # TODO check pkg_resource.require and see if it can replace this function + # TODO pkg_resource.require will also adjust the sys.path properly full_libpath = os.path.join(path, det_lib_path()) eggdir_regex = re.compile('%s-[0-9a-z.]+-py[0-9.]+.egg' % pkg.replace('-', '_')) @@ -82,13 +83,13 @@ def prep(path): def stage0(tmpdir): """STAGE 0: Prepare and install distribute via included (patched) distribute_setup.py script.""" - + info("\n\n+++ STAGE 0: installing distribute via included (patched) distribute_setup.py...\n\n") txt = DISTRIBUTE_SETUP_PY if not print_debug: # silence distribute_setup.py by redirecting output to /dev/null - txt = re.sub(r'([^\n]*)(return subprocess.call\(args)(\) == 0)', + txt = re.sub(r'([^\n]*)(return subprocess.call\(args)(\) == 0)', r"\1f = open(os.devnull, 'w'); \2, stdout=f, stderr=f\3", txt) # silence distribute_setup.py completely by setting high log level threshold @@ -109,7 +110,7 @@ def stage0(tmpdir): # install easy_install to temporary directory from distribute_setup import main as distribute_setup_main - orig_sys_argv = sys.argv + orig_sys_argv = sys.argv[:] # make a copy sys.argv.append('--prefix=%s' % tmpdir) distribute_setup_main() sys.argv = orig_sys_argv @@ -134,7 +135,7 @@ def stage1(tmpdir): """STAGE 1: temporary install EasyBuild.""" info("\n\n+++ STAGE 1: installing EasyBuild in temporary dir with easy_install...\n\n") - + from setuptools.command import easy_install # avoid having an 'easybuild' directory in the current dir @@ -142,17 +143,22 @@ def stage1(tmpdir): # prepare install dir targetdir_stage1 = os.path.join(tmpdir, 'eb_stage1') + # TODO prep does not cleanup the PATH?PYTHON/.. from stage0? + # TODO prep should restore the environement from an original copy of os.environ prep(targetdir_stage1) # set PATH, Python search path # install latest EasyBuild with easy_install from PyPi - cmd = '--always-copy --prefix=%s easybuild' % targetdir_stage1 + cmd = [] + cmd.append('--always-copy') + cmd.append('--prefix=%s easybuild' % targetdir_stage1) if not print_debug: - cmd = '--quiet ' + cmd - debug("installing EasyBuild with 'easy_install %s'" % cmd) - easy_install.main(cmd.split(' ')) - + cmd.insert(0, '--quiet') + debug("installing EasyBuild with 'easy_install %s'" % (" ".join(cmd))) + easy_install.main(cmd) + # figure out EasyBuild version # NOTE: we can't rely on importing VERSION, because other EasyBuild installations may be in sys.path + # TODO checking the output is not a garantee for anything (eg imagine the latest eb is already on your system, just not as module) version_re = re.compile("This is EasyBuild (?P[0-9.]*[a-z0-9]*) \(framework: [0-9.]*[a-z0-9]*, easyblocks: [0-9.]*[a-z0-9]*\)") version_out_file = os.path.join(tmpdir, 'eb_version.out') os.system("eb --version &> %s" % version_out_file) @@ -168,10 +174,10 @@ def stage1(tmpdir): for pkg in ['easyconfigs', 'easyblocks', 'framework']: pkg_egg_dir = find_egg_dir_for(targetdir_stage1, 'easybuild-%s' % pkg) - + # prepend EasyBuild egg dirs to sys.path, so we know which EasyBuild we're using sys.path.insert(0, pkg_egg_dir) - + # determine per-package versions based on egg dirs version_regex = re.compile('easybuild_%s-([0-9a-z.-]*)-py[0-9.]*.egg' % pkg) pkg_egg_dirname = os.path.basename(pkg_egg_dir) @@ -185,6 +191,7 @@ def stage1(tmpdir): # get rid of easy-install.pth file try: + # TODO if you don't want easy-install.pth, install with easy_install -m, and set all you need explicitly with pkg_resource.require() os.remove(os.path.join(targetdir_stage1, det_lib_path(), 'easy-install.pth')) except OSError, err: debug("Failed to remove easy-install.pth: %s" % err) @@ -195,7 +202,7 @@ def stage1(tmpdir): error("Found another easybuild-framework than expected: %s" % easybuild.framework.__file__) else: debug("Found easybuild-framework in expected path, good!") - + import easybuild.easyblocks if not tmpdir in easybuild.easyblocks.__file__: error("Found another easybuild-easyblocks than expected: %s" % easybuild.easyblocks.__file__) @@ -209,14 +216,14 @@ def stage2(tmpdir, versions, install_path): info("\n\n+++ STAGE 2: installing EasyBuild in temporary dir with EasyBuild from stage 1...\n\n") - # create easyconfig file + # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version']) f = open(ebfile, "w") f.write(EB_EC_FILE % versions) f.close() - - # set build path to tmp dir - os.environ['EASYBUILBUILDPATH'] = tmpdir + + # set build path to tmp dir + os.environ['EASYBUILDBUILDPATH'] = tmpdir # set install dir if install_path is not None: @@ -282,7 +289,7 @@ def main(): shutil.rmtree(tmpdir) info('Done!') - + if install_path is not None: info('EasyBuild v%s was installed to %s, so make sure your MODULEPATH includes %s' % \ (versions['version'], install_path, os.path.join(install_path, 'modules', 'all'))) @@ -290,7 +297,7 @@ def main(): info('EasyBuild v%s was installed to configured install path, make sure your MODULEPATH is set correctly.' % \ versions['version']) info('(default config => add "$HOME/.local/easybuild/modules/all" in MODULEPATH)') - + info("Run 'module load EasyBuild', and run 'eb --help' to get help on using EasyBuild.") # template easyconfig file for EasyBuild @@ -341,7 +348,7 @@ def main(): #+ if options.prefix_install: #+ install_args.append('--prefix=%s' % options.prefix_install) # return install_args -# +# # def _parse_args(): #@@ -529,6 +531,8 @@ # '--user', dest='user_install', action='store_true', default=False, diff --git a/easybuild/test/easyblocks_sandbox/easybuild/__init__.py b/easybuild/test/easyblocks_sandbox/easybuild/__init__.py new file mode 100644 index 0000000000..5b5529273b --- /dev/null +++ b/easybuild/test/easyblocks_sandbox/easybuild/__init__.py @@ -0,0 +1,4 @@ +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/__init__.py b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/__init__.py new file mode 100644 index 0000000000..5b5529273b --- /dev/null +++ b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/__init__.py @@ -0,0 +1,4 @@ +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/bar.py b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/bar.py new file mode 100644 index 0000000000..07f09506d7 --- /dev/null +++ b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/bar.py @@ -0,0 +1,46 @@ +## +# Copyright 2009-2013 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Generic EasyBuild support for building and installing bar, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM, MANDATORY + + +class bar(EasyBlock): + """Generic support for building/installing bar.""" + + @staticmethod + def extra_options(): + """Custom easyconfig parameters for bar.""" + + extra_vars = [ + ('bar_extra1', [None, "first bar-specific easyconfig parameter (mandatory)", MANDATORY]), + ('bar_extra2', ['BAR', "second bar-specific easyconfig parameter", CUSTOM]), + ] + return EasyBlock.extra_options(extra_vars) diff --git a/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foo.py b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foo.py new file mode 100644 index 0000000000..e03de5e9f8 --- /dev/null +++ b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foo.py @@ -0,0 +1,46 @@ +## +# Copyright 2009-2013 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing foo, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM, MANDATORY + + +class EB_foo(EasyBlock): + """Support for building/installing foo.""" + + @staticmethod + def extra_options(more_extra_vars=[]): + """Custom easyconfig parameters for foo.""" + + extra_vars = [ + ('foo_extra1', [None, "first foo-specific easyconfig parameter (mandatory)", MANDATORY]), + ('foo_extra2', ['FOO', "second foo-specific easyconfig parameter", CUSTOM]), + ] + return EasyBlock.extra_options(extra_vars + more_extra_vars) diff --git a/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foofoo.py b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foofoo.py new file mode 100644 index 0000000000..6fba26af16 --- /dev/null +++ b/easybuild/test/easyblocks_sandbox/easybuild/easyblocks/foofoo.py @@ -0,0 +1,46 @@ +## +# Copyright 2009-2013 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing foofoo, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.easyblocks.foo import EB_foo +from easybuild.framework.easyconfig import CUSTOM, MANDATORY + + +class EB_foofoo(EB_foo): + """Support for building/installing foofoo.""" + + @staticmethod + def extra_options(): + """Custom easyconfig parameters for foofoo.""" + + extra_vars = [ + ('foofoo_extra1', [None, "first foofoo-specific easyconfig parameter (mandatory)", MANDATORY]), + ('foofoo_extra2', ['FOOFOO', "second foofoo-specific easyconfig parameter", CUSTOM]), + ] + return EB_foo.extra_options(extra_vars) diff --git a/easybuild/test/options.py b/easybuild/test/options.py index eeb45ea6f6..b5e67cf5ab 100644 --- a/easybuild/test/options.py +++ b/easybuild/test/options.py @@ -35,7 +35,10 @@ from unittest import TestCase, TestLoader from unittest import main as unittestmain +import easybuild.easyblocks from easybuild.main import main +from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE +from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.tools.options import EasyBuildOptions from vsc import fancylogger @@ -276,7 +279,7 @@ def test_zzz_logtostdout(self): args = [ '--software-name=somethingrandom', - '--robot=.', + '--robot', '.', '--debug', stdout_arg, ] @@ -299,6 +302,64 @@ def test_zzz_logtostdout(self): fancylogger.logToFile(self.logfile) + def test_avail_easyconfig_params(self): + """Test listing available easyconfig parameters.""" + + def run_test(custom=None, extra_params=[]): + """Inner function to run actual test in current setting.""" + for avail_arg in [ + '-a', + '--avail-easyconfig-params', + ]: + + # clear log + open(self.logfile, 'w').write('') + + args = [ + avail_arg, + '--unittest-file=%s' % self.logfile, + ] + if custom is not None: + args.extend(['-e', custom]) + + try: + main((args, None)) + except: + pass + outtxt = open(self.logfile, 'r').read() + + # check whether all parameter types are listed + par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE, MANDATORY, MODULES, OTHER, TOOLCHAIN] + if custom is not None: + par_types.append(CUSTOM) + + for param_type in [x[1] for x in par_types]: + self.assertTrue(re.search("%s\n%s" % (param_type.upper(), '-'*len(param_type)), outtxt), + "Parameter type %s is featured in output of eb %s (args: %s): %s" % (param_type, avail_arg, args, outtxt)) + + # check a couple of easyconfig parameters + for param in ["name", "version", "toolchain", "versionsuffix", "makeopts", "sources", "start_dir", + "dependencies", "group", "exts_list", "moduleclass", "buildstats"] + extra_params: + self.assertTrue(re.search("%s:\s*\w.*" % param, outtxt), + "Parameter %s is listed with help in output of eb %s (args: %s): %s" % (param, avail_arg, args, outtxt)) + + # run test without checks for easyblock-custom easyconfig parameters + run_test() + + # also check whether available custom easyconfig parameters are listed + orig_sys_path = sys.path + + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'easyblocks_sandbox'))) + global easybuild + easybuild.easyblocks = reload(easybuild.easyblocks) + + run_test(custom='EB_foo', extra_params=['foo_extra1', 'foo_extra2']) + run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2']) + run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2']) + + # restore original Python search path + sys.path = orig_sys_path + def test_list_toolchains(self): """Test listing known compiler toolchains.""" @@ -317,6 +378,66 @@ def test_list_toolchains(self): for tc in ["dummy", "goalf", "ictce"]: self.assertTrue(re.search("%s: " % tc, outtxt), "Toolchain %s is included in list of known compiler toolchains") + def test_list_easyblocks(self): + """Test listing easyblock hierarchy.""" + + # adjust PYTHONPATH such that test easyblocks are found + orig_sys_path = sys.path + + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'easyblocks_sandbox'))) + global easybuild + easybuild.easyblocks = reload(easybuild.easyblocks) + + # simple view + for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']: + + # clear log + open(self.logfile, 'w').write('') + + args = [ + list_arg, + '--unittest-file=%s' % self.logfile, + ] + try: + main((args, None)) + except: + pass + outtxt = open(self.logfile, 'r').read() + + for pat in [ + r"EasyBlock\n", + r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n", + r"|--\s+bar\n", + ]: + + self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) + + # clear log + open(self.logfile, 'w').write('') + + # detailed view + args = [ + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + ] + try: + main((args, None)) + except: + pass + outtxt = open(self.logfile, 'r').read() + + for pat in [ + r"EasyBlock\s+\(easybuild.framework.easyblock\)\n", + r"|--\s+EB_foo\s+\(easybuild.easyblocks.foo\)\n|\s+|--\s+EB_foofoo\s+\(easybuild.easyblocks.foofoo\)\n", + r"|--\s+bar\s+\(easybuild.easyblocks.bar\)\n", + ]: + + self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) + + + # restore original Python search path + sys.path = orig_sys_path + def test_no_such_software(self): """Test using no arguments.""" diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index c4e6687935..f280520343 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -37,7 +37,7 @@ # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" # 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware! -VERSION = LooseVersion("1.2.0rc1") +VERSION = LooseVersion("1.3.0dev") UNKNOWN = "UNKNOWN" def get_git_revision():