diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 79eb82ac64..cd7ad8da81 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -48,11 +48,10 @@ from easybuild.framework.easyconfig.templates import template_documentation from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.extension import Extension -from easybuild.tools import build_log, config, run # @UnusedImport make sure config is always initialized! +from easybuild.tools import build_log, config, run # build_log should always stay there, to ensure EasyBuildLog from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL -from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS -from easybuild.tools.config import DEFAULT_PREFIX, DEFAULT_REPOSITORY +from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY from easybuild.tools.config import DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params @@ -63,9 +62,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.ordereddict import OrderedDict -import easybuild.tools.package.utilities as packaging -from easybuild.tools.package.utilities import DEFAULT_PNS -from easybuild.tools.package.activepns import avail_package_naming_scheme +from easybuild.tools.package.utilities import DEFAULT_PNS, avail_package_naming_schemes, check_pkg_support from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.version import this_is_easybuild @@ -267,7 +264,7 @@ def config_options(self): 'packagepath': ("The destination path for the packages built by package-tool", None, 'store', mk_full_default_path('packagepath')), 'package-naming-scheme': ("Packaging naming scheme choice", - 'choice', 'store', DEFAULT_PNS, sorted(avail_package_naming_scheme().keys())), + 'choice', 'store', DEFAULT_PNS, sorted(avail_package_naming_schemes().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), @@ -472,15 +469,12 @@ def postprocess(self): self._postprocess_config() - #Check experimental option dependencies (for now packaging) - #print "Got config_options: %s" % packaging.config_options - package_options = [ getattr(self.options, x) for x in packaging.config_options if getattr(self.options, x) ] - if any( package_options ): - packaging.option_postprocess() + # check whether packaging is supported when it's being used + if any([self.options.package_tool, self.options.package_type]): + check_pkg_support() else: self.log.debug("Didn't find any packaging options") - def _postprocess_external_modules_metadata(self): """Parse file(s) specifying metadata for external modules.""" # leave external_modules_metadata untouched if no files are provided diff --git a/easybuild/tools/package/activepns.py b/easybuild/tools/package/activepns.py deleted file mode 100644 index 7267bc014d..0000000000 --- a/easybuild/tools/package/activepns.py +++ /dev/null @@ -1,50 +0,0 @@ - - -from vsc.utils import fancylogger -from vsc.utils.missing import get_subclasses -from vsc.utils.patterns import Singleton -from easybuild.tools.config import get_package_naming_scheme -from easybuild.tools.build_log import EasyBuildError, print_error, print_msg -from easybuild.tools.package.packaging_naming_scheme.pns import PackagingNamingScheme -from easybuild.tools.utilities import import_available_modules - -def avail_package_naming_scheme(): - ''' - Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace - ''' - import_available_modules('easybuild.tools.package.packaging_naming_scheme') - - class_dict = dict([(x.__name__, x) for x in get_subclasses(PackagingNamingScheme)]) - - return class_dict - -class ActivePNS(object): - """ - The wrapper class for Package Naming Schmese, follows the model of Module Naming Schemes, mostly - """ - - __metaclass__ = Singleton - - def __init__(self, *args, **kwargs): - """Initialize logger and find available PNSes to load""" - self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) - - avail_pns = avail_package_naming_scheme() - sel_pns = get_package_naming_scheme() - if sel_pns in avail_pns: - self.pns = avail_pns[sel_pns]() - else: - raise EasyBuildError("Selected package naming scheme %s could not be found in %s", - sel_pns, avail_pns.keys()) - - def name(self, ec): - name = self.pns.name(ec) - return name - - def version(self, ec): - version = self.pns.version(ec) - return version - - def release(self, ec): - release = self.pns.release() - return release diff --git a/easybuild/tools/package/packaging_naming_scheme/easybuild_pns.py b/easybuild/tools/package/packaging_naming_scheme/easybuild_pns.py index 1dae429657..bc95cace8a 100644 --- a/easybuild/tools/package/packaging_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/packaging_naming_scheme/easybuild_pns.py @@ -1,10 +1,31 @@ - - - +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## """ -Default implementation of the EasyBuild packaging naming scheme +Implementation of the EasyBuild packaging naming scheme -@author: Rob Schmidt (Ottawa Hospital Research Institute) +@author: Robert Schmidt (Ottawa Hospital Research Institute) @author: Kenneth Hoste (Ghent University) """ @@ -17,6 +38,7 @@ class EasyBuildPNS(PackagingNamingScheme): REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] def name(self, ec): + """Determine package name""" self.log.debug("easyconfig dict for name looks like %s " % ec ) name_template = "eb%(eb_ver)s-%(name)s-%(version)s-%(toolchain)s" pkg_name = name_template % { @@ -28,10 +50,10 @@ def name(self, ec): return pkg_name def _toolchain(self, ec): + """Determine toolchain""" toolchain_template = "%(toolchain_name)s-%(toolchain_version)s" pkg_toolchain = toolchain_template % { 'toolchain_name': ec['toolchain']['name'], 'toolchain_version': ec['toolchain']['version'], } return pkg_toolchain - diff --git a/easybuild/tools/package/packaging_naming_scheme/pns.py b/easybuild/tools/package/packaging_naming_scheme/pns.py index 408a1b6904..b1569d0f66 100644 --- a/easybuild/tools/package/packaging_naming_scheme/pns.py +++ b/easybuild/tools/package/packaging_naming_scheme/pns.py @@ -1,28 +1,55 @@ - +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## + +""" +General package naming scheme. + +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Kenneth Hoste (Ghent University) +""" from vsc.utils import fancylogger from easybuild.tools.config import build_option from easybuild.tools.version import VERSION as EASYBUILD_VERSION -options = [ "package-naming-name-template", "package-naming-version-template", "package-naming-toolchain-template" ] -class PackagingNamingScheme(object): - """Abstract class for package naming scheme""" +class PackagingNamingScheme(object): + """Abstract class for package naming schemes""" - def __init__(self, *args, **kwargs): + def __init__(self): """initialize logger.""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.eb_ver = EASYBUILD_VERSION - def name(self,ec): - """Return name of the package, by default would include name, version, toolchain""" + def name(self, ec): + """Determine package name""" raise NotImplementedError - - def version(self,ec): - """The version in the version part of the package""" - return ec['version'] - - def release(self,ec=None): - """Just the release""" - return build_option('package_release') + def version(self, ec): + """Determine package version""" + return ec['version'] + def release(self, ec=None): + """Determine package release""" + return build_option('package_release') diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index ccaef6bd38..14e215e2ca 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ -# # -# Copyright 2009-2014 Ghent University +## +# Copyright 2015-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -21,10 +21,10 @@ # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see . -# # +## """ -A place for packaging functions +Various utilities related to packaging support. @author: Marc Litherland @author: Gianluca Santarossa (Novartis) @@ -32,97 +32,134 @@ @author: Fotis Georgatos (Uni.Lu, NTUA) @author: Kenneth Hoste (Ghent University) """ - import os import tempfile import pprint from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses +from vsc.utils.patterns import Singleton -from easybuild.tools.run import run_cmd -from easybuild.tools.config import build_option -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.config import get_package_naming_scheme from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import which -from easybuild.tools.package.activepns import ActivePNS +from easybuild.tools.package.packaging_naming_scheme.pns import PackagingNamingScheme +from easybuild.tools.run import run_cmd +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.utilities import import_available_modules + DEFAULT_PNS = 'EasyBuildPNS' -_log = fancylogger.getLogger('tools.packaging') -# This is an abbreviated list of the package options, eventually it might make sense to set them -# all in the "plugin" rather than in tools.options -config_options = [ 'package_tool', 'package_type' ] +_log = fancylogger.getLogger('tools.package') + -def package_fpm(easyblock, modfile_path, package_type="rpm" ): - ''' +def avail_package_naming_schemes(): + """ + Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace + """ + import_available_modules('easybuild.tools.package.packaging_naming_scheme') + class_dict = dict([(x.__name__, x) for x in get_subclasses(PackagingNamingScheme)]) + return class_dict + + +def package_fpm(easyblock, modfile_path, package_type='rpm'): + """ This function will build a package using fpm and return the directory where the packages are - ''' - - workdir = tempfile.mkdtemp() - _log.info("Will be writing RPM to %s" % workdir) + """ + workdir = tempfile.mkdtemp(prefix='eb-pkgs') + _log.info("Will be creating packages in %s", workdir) try: os.chdir(workdir) except OSError, err: - raise EasyBuildError("Failed to chdir into workdir: %s : %s", workdir, err) + raise EasyBuildError("Failed to chdir into workdir %s: %s", workdir, err) package_naming_scheme = ActivePNS() pkgname = package_naming_scheme.name(easyblock.cfg) - pkgver = package_naming_scheme.version(easyblock.cfg) - pkgrel = package_naming_scheme.release(easyblock.cfg) + pkgver = package_naming_scheme.version(easyblock.cfg) + pkgrel = package_naming_scheme.release(easyblock.cfg) - _log.debug("Got the pns values for (name, version, release): (%s, %s, %s)" % (pkgname, pkgver, pkgrel)) + _log.debug("Got the PNS values for (name, version, release): (%s, %s, %s)", pkgname, pkgver, pkgrel) deps = [] if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME: toolchain_dict = easyblock.toolchain.as_dict() deps.extend([toolchain_dict]) deps.extend(easyblock.cfg.dependencies()) - - _log.debug("The dependencies to be added to the package are: " + pprint.pformat([easyblock.toolchain.as_dict()]+easyblock.cfg.dependencies())) - depstring = "" + + _log.debug("The dependencies to be added to the package are: %s", + pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies())) + depstring = '' for dep in deps: - _log.debug("The dep added looks like %s " % dep) + _log.debug("The dep added looks like %s ", dep) dep_pkgname = package_naming_scheme.name(dep) - depstring += " --depends '%s'" % ( dep_pkgname) + depstring += " --depends '%s'" % dep_pkgname - cmdlist=[ + cmdlist = [ 'fpm', '--workdir', workdir, '--name', pkgname, '--provides', pkgname, - '-t', package_type, # target - '-s', 'dir', # source + '-t', package_type, # target + '-s', 'dir', # source '--version', pkgver, '--iteration', pkgrel, - ] - cmdlist.extend([ depstring ]) - cmdlist.extend([ + depstring, easyblock.installdir, - modfile_path - ]) - cmdstr = " ".join(cmdlist) - _log.debug("The flattened cmdlist looks like" + cmdstr) - out = run_cmd(cmdstr, log_all=True, simple=True) - - _log.info("wrote rpm to %s" % (workdir) ) + modfile_path, + ] + cmd = ' '.join(cmdlist) + _log.debug("The flattened cmdlist looks like: %s", cmd) + run_cmd(cmd, log_all=True, simple=True) + + _log.info("Created %s package in %s", package_type, workdir) return workdir -def option_postprocess(): - ''' - Called from easybuild.tools.options.postprocess to check that experimental is triggered and fpm is available - ''' +def check_pkg_support(): + """Check whether packaging is supported, i.e. whether the required dependencies are available.""" - _log.experimental("Using the packaging module, This is experimental") + _log.experimental("Support for packaging installed software.") fpm_path = which('fpm') rpmbuild_path = which('rpmbuild') if fpm_path and rpmbuild_path: - _log.info("fpm found at: %s" % fpm_path) + _log.info("fpm found at: %s", fpm_path) else: raise EasyBuildError("Need both fpm and rpmbuild. Found fpm: %s rpmbuild: %s", fpm_path, rpmbuild_path) +class ActivePNS(object): + """ + The wrapper class for Package Naming Schemes. + """ + __metaclass__ = Singleton + + def __init__(self): + """Initialize logger and find available PNSes to load""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + avail_pns = avail_package_naming_schemes() + sel_pns = get_package_naming_scheme() + if sel_pns in avail_pns: + self.pns = avail_pns[sel_pns]() + else: + raise EasyBuildError("Selected package naming scheme %s could not be found in %s", + sel_pns, avail_pns.keys()) + + def name(self, ec): + """Determine package name""" + name = self.pns.name(ec) + return name + + def version(self, ec): + """Determine package version""" + version = self.pns.version(ec) + return version + + def release(self, ec): + """Determine package release""" + release = self.pns.release() + return release diff --git a/test/framework/package.py b/test/framework/package.py new file mode 100644 index 0000000000..8b28dc0eef --- /dev/null +++ b/test/framework/package.py @@ -0,0 +1,79 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for packaging support. + +@author: Kenneth Hoste (Ghent University) +""" +import os +import stat + +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader +from unittest import main as unittestmain + +import easybuild.tools.build_log +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import adjust_permissions, write_file +from easybuild.tools.package.utilities import avail_package_naming_schemes, check_pkg_support + + +class PackageTest(EnhancedTestCase): + """Tests for packaging support.""" + + def test_avail_package_naming_schemes(self): + """Test avail_package_naming_schemes()""" + self.assertEqual(sorted(avail_package_naming_schemes().keys()), ['EasyBuildPNS']) + + def test_check_pkg_support(self): + """Test check_pkg_support().""" + # hard enable experimental + orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + easybuild.tools.build_log.EXPERIMENTAL = True + + # clear $PATH to make sure fpm/rpmbuild can not be found + os.environ['PATH'] = '' + + self.assertErrorRegex(EasyBuildError, "Need both fpm and rpmbuild", check_pkg_support) + + for binary in ['fpm', 'rpmbuild']: + binpath = os.path.join(self.test_prefix, binary) + write_file(binpath, '#!/bin/bash') + adjust_permissions(binpath, stat.S_IXUSR, add=True) + os.environ['PATH'] = self.test_prefix + + check_pkg_support() + + # restore + easybuild.tools.build_log.EXPERIMENTAL = orig_experimental + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(PackageTest) + + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/suite.py b/test/framework/suite.py index 2b6001221f..636966391a 100644 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -72,6 +72,7 @@ import test.framework.modulestool as mt import test.framework.options as o import test.framework.parallelbuild as p +import test.framework.package as pkg import test.framework.repository as r import test.framework.robot as robot import test.framework.run as run @@ -100,7 +101,8 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p] +tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, + p, pkg] SUITE = unittest.TestSuite([x.suite() for x in tests])