diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b2828a6d41..a9a338794e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -59,7 +59,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, print_error, print_msg from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath -from easybuild.tools.config import install_path, log_path, read_only_installdir, source_paths +from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env from easybuild.tools.filetools import DEFAULT_CHECKSUM from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name @@ -68,9 +68,10 @@ from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator -from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import get_software_root, modules_tool +from easybuild.tools.package.utilities import package from easybuild.tools.repository.repository import init_repository from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.systemtools import det_parallelism, use_group @@ -86,6 +87,7 @@ MODULE_STEP = 'module' PACKAGE_STEP = 'package' PATCH_STEP = 'patch' +PERMISSIONS_STEP = 'permissions' POSTPROC_STEP = 'postproc' PREPARE_STEP = 'prepare' READY_STEP = 'ready' @@ -1490,15 +1492,34 @@ def extensions_step(self, fetch=False): self.clean_up_fake_module(fake_mod_data) def package_step(self): - """Package software (e.g. into an RPM).""" - pass + """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" + + if build_option('package'): + + pkgtype = build_option('package_type') + pkgdir_dest = os.path.abspath(package_path()) + opt_force = build_option('force') + + self.log.info("Generating %s package in %s", pkgtype, pkgdir_dest) + pkgdir_src = package(self) + + mkdir(pkgdir_dest) + + for src_file in glob.glob(os.path.join(pkgdir_src, "*.%s" % pkgtype)): + dest_file = os.path.join(pkgdir_dest, os.path.basename(src_file)) + if os.path.exists(dest_file) and not opt_force: + raise EasyBuildError("Unable to copy package %s to %s (already exists).", src_file, dest_file) + else: + self.log.info("Copied package %s to %s", src_file, pkgdir_dest) + shutil.copy(src_file, pkgdir_dest) + + else: + self.log.info("Skipping package step (not enabled)") def post_install_step(self): """ Do some postprocessing - run post install commands if any were specified - - set file permissions .... - Installing user must be member of the group that it is changed to """ if self.cfg['postinstallcmds'] is not None: # make sure we have a list of commands @@ -1509,27 +1530,6 @@ def post_install_step(self): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) - if self.group is not None: - # remove permissions for others, and set group ID - try: - perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1], - relative=True, ignore_errors=True) - except EasyBuildError, err: - raise EasyBuildError("Unable to change group permissions of file(s): %s", err) - self.log.info("Successfully made software only available for group %s (gid %s)" % self.group) - - if read_only_installdir(): - # remove write permissions for everyone - perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) - self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.") - else: - # remove write permissions for group and other to protect installation - perms = stat.S_IWGRP | stat.S_IWOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) - self.log.info("Successfully removed write permissions recursively for group/other on install dir.") - def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False): """ Do a sanity check on the installation @@ -1715,6 +1715,39 @@ def make_module_step(self, fake=False): return modpath + def permissions_step(self): + """ + Finalize installation procedure: adjust permissions as configured, change group ownership (if requested). + Installing user must be member of the group that it is changed to. + """ + if self.group is not None: + # remove permissions for others, and set group ID + try: + perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1], + relative=True, ignore_errors=True) + except EasyBuildError, err: + raise EasyBuildError("Unable to change group permissions of file(s): %s", err) + self.log.info("Successfully made software only available for group %s (gid %s)" % self.group) + + if build_option('read_only_installdir'): + # remove write permissions for everyone + perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.") + + elif build_option('group_writable_installdir'): + # enable write permissions for group + perms = stat.S_IWGRP + adjust_permissions(self.installdir, perms, add=True, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully enabled write permissions recursively for group on install dir.") + + else: + # remove write permissions for group and other + perms = stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully removed write permissions recursively for group/other on install dir.") + def test_cases_step(self): """ Run provided test cases. @@ -1863,11 +1896,12 @@ def prepare_step_spec(initial): # part 3: post-iteration part steps_part3 = [ (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step()], False), - (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], True), (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step()], True), (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step()], False), (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step()], False), (MODULE_STEP, 'creating module', [lambda x: x.make_module_step()], False), + (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step()], False), + (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], False), ] # full list of steps, included iterated steps @@ -1981,6 +2015,9 @@ def build_and_install_one(ecdict, init_env): new_log_dir = os.path.dirname(app.logfile) else: new_log_dir = os.path.join(app.installdir, config.log_path()) + if build_option('read_only_installdir'): + # temporarily re-enable write permissions for copying log/easyconfig to install dir + adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=False) # collect build stats _log.info("Collecting build stats...") @@ -2022,6 +2059,10 @@ def build_and_install_one(ecdict, init_env): except (IOError, OSError), err: print_error("Failed to copy easyconfig %s to %s: %s" % (spec, newspec, err)) + if build_option('read_only_installdir'): + # take away user write permissions (again) + adjust_permissions(new_log_dir, stat.S_IWUSR, add=False, recursive=False) + # build failed else: success = False diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b96ac2c5a6..d34412a9b5 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -422,9 +422,9 @@ def dependencies(self): # if filter-deps option is provided we "clean" the list of dependencies for # each processed easyconfig to remove the unwanted dependencies + self.log.debug("Dependencies BEFORE filtering: %s" % deps) filter_deps = build_option('filter_deps') if filter_deps: - self.log.debug("Dependencies BEFORE filtering: %s" % deps) filtered_deps = [] for dep in deps: if dep['name'] not in filter_deps: diff --git a/easybuild/main.py b/easybuild/main.py index c0f90b4b65..bfa223975c 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -37,6 +37,7 @@ """ import copy import os +import stat import sys import tempfile import traceback @@ -53,9 +54,10 @@ from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import get_repository, get_repositorypath -from easybuild.tools.filetools import cleanup, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, write_file from easybuild.tools.options import process_software_build_specs from easybuild.tools.robot import det_robot_path, dry_run, resolve_dependencies, search_easyconfigs +from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_module_list, session_state @@ -129,7 +131,14 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if 'log_file' in ec_res: test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1]) - write_file(test_report_fp, test_report_txt) + parent_dir = os.path.dirname(test_report_fp) + # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used + if os.stat(parent_dir).st_mode & 0200: + write_file(test_report_fp, test_report_txt) + else: + adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) + write_file(test_report_fp, test_report_txt) + adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: @@ -210,6 +219,12 @@ def main(testing_data=(None, None, None)): config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) + # check whether packaging is supported when it's being used + if options.package: + check_pkg_support() + else: + _log.debug("Packaging not enabled, so not checking for packaging support.") + # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index a68d56a908..ad30ff312c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -52,6 +52,10 @@ _log = fancylogger.getLogger('config', fname=False) +PKG_TOOL_FPM = 'fpm' +PKG_TYPE_RPM = 'rpm' + + DEFAULT_JOB_BACKEND = 'PbsPython' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MNS = 'EasyBuildMNS' @@ -60,16 +64,20 @@ DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', 'installpath': '', + 'packagepath': 'packages', 'repositorypath': 'ebfiles_repo', 'sourcepath': 'sources', 'subdir_modules': 'modules', 'subdir_software': 'software', } +DEFAULT_PKG_RELEASE = '1' +DEFAULT_PKG_TOOL = PKG_TOOL_FPM +DEFAULT_PKG_TYPE = PKG_TYPE_RPM +DEFAULT_PNS = 'EasyBuildPNS' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' DEFAULT_STRICT = run.WARN - # utility function for obtaining default paths def mk_full_default_path(name, prefix=DEFAULT_PREFIX): """Create full path, avoid '/' at the end.""" @@ -115,8 +123,11 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'debug', 'experimental', 'force', + 'group_writable_installdir', 'hidden', 'module_only', + 'package', + 'read_only_installdir', 'robot', 'sequential', 'set_gid_bit', @@ -131,6 +142,15 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_STRICT: [ 'strict', ], + DEFAULT_PKG_RELEASE: [ + 'package_release', + ], + DEFAULT_PKG_TOOL: [ + 'package_tool', + ], + DEFAULT_PKG_TYPE: [ + 'package_type', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { @@ -203,6 +223,8 @@ class ConfigurationVariables(FrozenDictKnownKeys): 'module_naming_scheme', 'module_syntax', 'modules_tool', + 'packagepath', + 'package_naming_scheme', 'prefix', 'repository', 'repositorypath', @@ -371,6 +393,20 @@ def get_repositorypath(): return ConfigurationVariables()['repositorypath'] +def get_package_naming_scheme(): + """ + Return the package naming scheme + """ + return ConfigurationVariables()['package_naming_scheme'] + + +def package_path(): + """ + Return the path where built packages are copied to + """ + return ConfigurationVariables()['packagepath'] + + def get_modules_tool(): """ Return modules tool (EnvironmentModulesC, Lmod, ...) @@ -464,16 +500,6 @@ def get_log_filename(name, version, add_salt=False): return filepath -def read_only_installdir(): - """ - Return whether installation dir should be fully read-only after installation. - """ - # FIXME (see issue #123): add a config option to set this, should be True by default (?) - # this also needs to be checked when --force is used; - # install dir will have to (temporarily) be made writeable again for owner in that case - return False - - def module_classes(): """ Return list of module classes specified in config file. diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5745cd6cbe..832ef5eca7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -52,7 +52,8 @@ from easybuild.tools import build_log, run # build_log should always stay there, to ensure EasyBuildLog from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX -from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX +from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS +from easybuild.tools.config import DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_STRICT from easybuild.tools.config import get_pretend_installpath, mk_full_default_path, set_tmpdir from easybuild.tools.configobj import ConfigObj, ConfigObjError @@ -66,6 +67,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.version import this_is_easybuild @@ -203,6 +205,8 @@ def override_options(self): 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), + 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", + None, 'store_true', False), 'hidden': ("Install 'hidden' module file(s) by prefixing their name with '.'", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " @@ -216,6 +220,8 @@ def override_options(self): None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), None, 'store_true', False, 'p'), + 'read-only-installdir': ("Set read-only permissions on installation directory after installation", + None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), @@ -271,6 +277,10 @@ def config_options(self): None, 'store_or_None', None, {'metavar': "PATH"}), 'modules-tool': ("Modules tool to use", 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), + 'packagepath': ("The destination path for the packages built by package-tool", + None, 'store', mk_full_default_path('packagepath')), + 'package-naming-scheme': ("Packaging naming scheme choice", + 'choice', 'store', DEFAULT_PNS, sorted(avail_package_naming_schemes().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), @@ -357,6 +367,20 @@ def regtest_options(self): self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) + def package_options(self): + # package-related options + descr = ("Package options", "Control packaging performed by EasyBuild.") + + opts = OrderedDict({ + 'package': ("Enabling packaging", None, 'store_true', False), + 'package-tool': ("Packaging tool to use", None, 'store', DEFAULT_PKG_TOOL), + 'package-type': ("Type of package to generate", None, 'store', DEFAULT_PKG_TYPE), + 'package-release': ("Package release iteration number", None, 'store', DEFAULT_PKG_RELEASE), + }) + + self.log.debug("package_options: descr %s opts %s" % (descr, opts)) + self.add_group_parser(opts, descr) + def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") @@ -535,7 +559,7 @@ def _postprocess_config(self): if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself - for dest in ['installpath', 'buildpath', 'sourcepath', 'repository', 'repositorypath']: + for dest in ['installpath', 'buildpath', 'sourcepath', 'repository', 'repositorypath', 'packagepath']: if not self.options._action_taken.get(dest, False): if dest == 'repository': setattr(self.options, dest, DEFAULT_REPOSITORY) diff --git a/easybuild/tools/package/__init__.py b/easybuild/tools/package/__init__.py new file mode 100644 index 0000000000..51a62d3f44 --- /dev/null +++ b/easybuild/tools/package/__init__.py @@ -0,0 +1,38 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.package submodule of EasyBuild, +which contains support for packaging and package naming schemes that can be overriden to cover site customizations. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/package/package_naming_scheme/__init__.py b/easybuild/tools/package/package_naming_scheme/__init__.py new file mode 100644 index 0000000000..6af2449881 --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/__init__.py @@ -0,0 +1,37 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.package.package_naming_scheme submodule of EasyBuild. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py new file mode 100644 index 0000000000..d6698b551e --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -0,0 +1,53 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of the EasyBuild packaging naming scheme + +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme +from easybuild.tools.version import VERSION as EASYBUILD_VERSION + + +class EasyBuildPNS(PackageNamingScheme): + """Class implmenting the default EasyBuild packaging naming scheme.""" + + def name(self, ec): + """Determine package name""" + self.log.debug("Easyconfig dict passed to name() looks like: %s ", ec) + return '%s-%s' % (ec['name'], det_full_ec_version(ec)) + + def version(self, ec): + """Determine package version: EasyBuild version used to build & install.""" + ebver = str(EASYBUILD_VERSION) + if ebver.endswith('dev'): + # try and make sure that 'dev' EasyBuild version is not considered newer just because it's longer + # (e.g., 2.2.0 vs 2.2.0dev) + # cfr. http://rpm.org/ticket/56, + # https://debian-handbook.info/browse/stable/sect.manipulating-packages-with-dpkg.html (see box in 5.4.3) + ebver.replace('dev', '~dev') + return 'eb-%s' % ebver diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py new file mode 100644 index 0000000000..d66bce2262 --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -0,0 +1,58 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## + +""" +Abstract implementation of a package naming scheme. + +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Kenneth Hoste (Ghent University) +""" +from abc import ABCMeta, abstractmethod +from vsc.utils import fancylogger + +from easybuild.tools.config import build_option + + +class PackageNamingScheme(object): + """Abstract class for package naming schemes""" + __metaclass__ = ABCMeta + + def __init__(self): + """initialize logger.""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + @abstractmethod + def name(self, ec): + """Determine package name""" + pass + + @abstractmethod + def version(self, ec): + """Determine package version.""" + pass + + def release(self, ec=None): + """Determine package release""" + return build_option('package_release') diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py new file mode 100644 index 0000000000..a3d559bcc8 --- /dev/null +++ b/easybuild/tools/package/utilities.py @@ -0,0 +1,193 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## + +""" +Various utilities related to packaging support. + +@author: Marc Litherland (Novartis) +@author: Gianluca Santarossa (Novartis) +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Fotis Georgatos (Uni.Lu, NTUA) +@author: Kenneth Hoste (Ghent University) +""" +import os +import tempfile +import pprint + +from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses +from vsc.utils.patterns import Singleton + +from easybuild.tools.config import PKG_TOOL_FPM, PKG_TYPE_RPM, build_option, get_package_naming_scheme +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import which +from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme +from easybuild.tools.run import run_cmd +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.utilities import import_available_modules + + +_log = fancylogger.getLogger('tools.package') + + +def avail_package_naming_schemes(): + """ + Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace + """ + import_available_modules('easybuild.tools.package.package_naming_scheme') + class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)]) + return class_dict + + +def package(easyblock): + """ + Package installed software, according to active packaging configuration settings.""" + pkgtool = build_option('package_tool') + + if pkgtool == PKG_TOOL_FPM: + pkgdir = package_with_fpm(easyblock) + else: + raise EasyBuildError("Unknown packaging tool specified: %s", pkgtool) + + return pkgdir + + +def package_with_fpm(easyblock): + """ + This function will build a package using fpm and return the directory where the packages are + """ + workdir = tempfile.mkdtemp(prefix='eb-pkgs-') + pkgtype = build_option('package_type') + _log.info("Will be creating %s package(s) in %s", pkgtype, workdir) + + try: + origdir = os.getcwd() + os.chdir(workdir) + except OSError, err: + raise EasyBuildError("Failed to chdir into workdir %s: %s", workdir, err) + + package_naming_scheme = ActivePNS() + + pkgname = package_naming_scheme.name(easyblock.cfg) + pkgver = package_naming_scheme.version(easyblock.cfg) + pkgrel = package_naming_scheme.release(easyblock.cfg) + + _log.debug("Got the PNS values for (name, version, release): (%s, %s, %s)", pkgname, pkgver, pkgrel) + deps = [] + if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME: + toolchain_dict = easyblock.toolchain.as_dict() + deps.extend([toolchain_dict]) + + deps.extend(easyblock.cfg.dependencies()) + + _log.debug("The dependencies to be added to the package are: %s", + pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies())) + depstring = '' + for dep in deps: + _log.debug("The dep added looks like %s ", dep) + dep_pkgname = package_naming_scheme.name(dep) + depstring += " --depends '%s'" % dep_pkgname + + cmdlist = [ + PKG_TOOL_FPM, + '--workdir', workdir, + '--name', pkgname, + '--provides', pkgname, + '-t', pkgtype, # target + '-s', 'dir', # source + '--version', pkgver, + '--iteration', pkgrel, + depstring, + easyblock.installdir, + easyblock.module_generator.filename, + ] + cmd = ' '.join(cmdlist) + _log.debug("The flattened cmdlist looks like: %s", cmd) + run_cmd(cmd, log_all=True, simple=True) + + _log.info("Created %s package(s) in %s", pkgtype, workdir) + + try: + os.chdir(origdir) + except OSError, err: + raise EasyBuildError("Failed to chdir back to %s: %s", origdir, err) + + return workdir + + +def check_pkg_support(): + """Check whether packaging is supported, i.e. whether the required dependencies are available.""" + # packaging support is considered experimental for now (requires using --experimental) + _log.experimental("Support for packaging installed software.") + + pkgtool = build_option('package_tool') + pkgtool_path = which(pkgtool) + if pkgtool_path: + _log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path) + + # rpmbuild is required for generating RPMs with FPM + if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM: + rpmbuild_path = which('rpmbuild') + if rpmbuild_path: + _log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path) + else: + raise EasyBuildError("rpmbuild is required when generating RPM packages with FPM, but was not found") + + else: + raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool) + + +class ActivePNS(object): + """ + The wrapper class for Package Naming Schemes. + """ + __metaclass__ = Singleton + + def __init__(self): + """Initialize logger and find available PNSes to load""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + avail_pns = avail_package_naming_schemes() + sel_pns = get_package_naming_scheme() + if sel_pns in avail_pns: + self.pns = avail_pns[sel_pns]() + else: + raise EasyBuildError("Selected package naming scheme %s could not be found in %s", + sel_pns, avail_pns.keys()) + + def name(self, ec): + """Determine package name""" + name = self.pns.name(ec) + return name + + def version(self, ec): + """Determine package version""" + version = self.pns.version(ec) + return version + + def release(self, ec): + """Determine package release""" + release = self.pns.release() + return release diff --git a/setup.py b/setup.py index edf1ff7655..cb9af5cc78 100644 --- a/setup.py +++ b/setup.py @@ -73,8 +73,9 @@ def find_rel_test(): "easybuild", "easybuild.framework", "easybuild.framework.easyconfig", "easybuild.framework.easyconfig.format", "easybuild.toolchains", "easybuild.toolchains.compiler", "easybuild.toolchains.mpi", "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", "easybuild.tools.deprecated", - "easybuild.tools.job", "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", "easybuild.tools.repository", - "test.framework", "test", + "easybuild.tools.job", "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", + "easybuild.tools.package", "easybuild.tools.package.package_naming_scheme", + "easybuild.tools.repository", "test.framework", "test", ] setup( diff --git a/test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb b/test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb similarity index 97% rename from test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb rename to test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb index 632c832da3..b866a7ee1c 100644 --- a/test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb +++ b/test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb @@ -1,5 +1,6 @@ name = 'toy' version = '0.0' +versionsuffix = '-test' homepage = 'http://hpcugent.github.com/easybuild' description = "Toy C program." diff --git a/test/framework/package.py b/test/framework/package.py new file mode 100644 index 0000000000..53feaf504f --- /dev/null +++ b/test/framework/package.py @@ -0,0 +1,143 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for packaging support. + +@author: Kenneth Hoste (Ghent University) +""" +import os +import stat + +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader +from unittest import main as unittestmain + +import easybuild.tools.build_log +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import adjust_permissions, write_file +from easybuild.tools.package.utilities import ActivePNS, avail_package_naming_schemes, check_pkg_support, package +from easybuild.tools.version import VERSION as EASYBUILD_VERSION + + +MOCKED_FPM = """#!/bin/bash +# only parse what we need to spit out the expected package file, ignore the rest +workdir=`echo $@ | sed 's/--workdir \([^ ]*\).*/\\1/g'` +name=`echo $@ | sed 's/.* --name \([^ ]*\).*/\\1/g'` +version=`echo $@ | sed 's/.*--version \([^ ]*\).*/\\1/g'` +iteration=`echo $@ | sed 's/.*--iteration \([^ ]*\).*/\\1/g'` +target=`echo $@ | sed 's/.*-t \([^ ]*\).*/\\1/g'` + +echo "thisisan$target" > ${workdir}/${name}-${version}.${iteration}.${target} +""" + + +def mock_fpm(tmpdir): + """Put mocked version of fpm command in place in specified tmpdir.""" + # put mocked 'fpm' command in place, just for testing purposes + fpm = os.path.join(tmpdir, 'fpm') + write_file(fpm, MOCKED_FPM) + adjust_permissions(fpm, stat.S_IXUSR, add=True) + + # also put mocked rpmbuild in place + rpmbuild = os.path.join(tmpdir, 'rpmbuild') + write_file(rpmbuild, '#!/bin/bash') # only needs to be there, doesn't need to actually do something... + adjust_permissions(rpmbuild, stat.S_IXUSR, add=True) + + os.environ['PATH'] = '%s:%s' % (tmpdir, os.environ['PATH']) + + +class PackageTest(EnhancedTestCase): + """Tests for packaging support.""" + + def test_avail_package_naming_schemes(self): + """Test avail_package_naming_schemes()""" + self.assertEqual(sorted(avail_package_naming_schemes().keys()), ['EasyBuildPNS']) + + def test_check_pkg_support(self): + """Test check_pkg_support().""" + # hard enable experimental + orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + easybuild.tools.build_log.EXPERIMENTAL = True + + # clear $PATH to make sure fpm/rpmbuild can not be found + os.environ['PATH'] = '' + + self.assertErrorRegex(EasyBuildError, "Selected packaging tool 'fpm' not found", check_pkg_support) + + for binary in ['fpm', 'rpmbuild']: + binpath = os.path.join(self.test_prefix, binary) + write_file(binpath, '#!/bin/bash') + adjust_permissions(binpath, stat.S_IXUSR, add=True) + os.environ['PATH'] = self.test_prefix + + # no errors => support check passes + check_pkg_support() + + # restore + easybuild.tools.build_log.EXPERIMENTAL = orig_experimental + + def test_active_pns(self): + """Test use of ActivePNS.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.6.4.eb'), validate=False) + + pns = ActivePNS() + + # default: EasyBuild package naming scheme, pkg release 1 + self.assertEqual(pns.name(ec), 'OpenMPI-1.6.4-GCC-4.6.4') + self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION) + self.assertEqual(pns.release(ec), '1') + + def test_package(self): + """Test package function.""" + init_config(build_options={'silent': True}) + + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) + + mock_fpm(self.test_prefix) + + # import needs to be done here, since test easyblocks are only included later + from easybuild.easyblocks.toy import EB_toy + easyblock = EB_toy(ec) + + # build & install first + easyblock.run_all_steps(False) + + # package using default packaging configuration (FPM to build RPM packages) + pkgdir = package(easyblock) + + pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) + self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(PackageTest) + + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/suite.py b/test/framework/suite.py index 9e96ec2dee..34caa3ec8f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -74,6 +74,7 @@ import test.framework.modulestool as mt import test.framework.options as o import test.framework.parallelbuild as p +import test.framework.package as pkg import test.framework.repository as r import test.framework.robot as robot import test.framework.run as run @@ -102,7 +103,8 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p, i, d] +tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, + p, i, pkg, d] SUITE = unittest.TestSuite([x.suite() for x in tests]) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 46797db2d1..2aa8469112 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -36,6 +36,7 @@ import sys import tempfile from test.framework.utilities import EnhancedTestCase +from test.framework.package import mock_fpm from unittest import TestLoader from unittest import main as unittestmain from vsc.utils.fancylogger import setLogLevelDebug, logToScreen @@ -44,8 +45,9 @@ from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax -from easybuild.tools.filetools import mkdir, read_file, which, write_file +from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, which, write_file from easybuild.tools.modules import modules_tool +from easybuild.tools.version import VERSION as EASYBUILD_VERSION class ToyBuildTest(EnhancedTestCase): @@ -480,6 +482,31 @@ def test_toy_permissions(self): # restore original umask os.umask(orig_umask) + def test_toy_permissions_installdir(self): + """Test --read-only-installdir and --group-write-installdir.""" + # set umask hard to verify default reliably + orig_umask = os.umask(0022) + + self.test_toy_build() + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0755, "%s has default permissions" % self.test_installpath) + shutil.rmtree(self.test_installpath) + + self.test_toy_build(extra_args=['--read-only-installdir']) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0555, "%s has read-only permissions" % self.test_installpath) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy')).st_mode & 0777 + self.assertEqual(installdir_perms, 0755, "%s has default permissions" % self.test_installpath) + adjust_permissions(os.path.join(self.test_installpath, 'software', 'toy', '0.0'), stat.S_IWUSR, add=True) + shutil.rmtree(self.test_installpath) + + self.test_toy_build(extra_args=['--group-writable-installdir']) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0775, "%s has group write permissions" % self.test_installpath) + + # restore original umask + os.umask(orig_umask) + def test_toy_gid_sticky_bits(self): """Test setting gid and sticky bits.""" subdirs = [ @@ -693,8 +720,8 @@ def test_toy_advanced(self): """Test toy build with extensions and non-dummy toolchain.""" test_dir = os.path.abspath(os.path.dirname(__file__)) os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - test_ec = os.path.join(test_dir, 'easyconfigs', 'toy-0.0-gompi-1.3.12.eb') - self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-1.3.12') + test_ec = os.path.join(test_dir, 'easyconfigs', 'toy-0.0-gompi-1.3.12-test.eb') + self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-1.3.12-test') def test_toy_hidden(self): """Test installing a hidden module.""" @@ -970,6 +997,39 @@ def test_module_only(self): modtxt = read_file(toy_mod + '.lua') self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + def test_package(self): + """Test use of --package and accompanying package configuration settings.""" + mock_fpm(self.test_prefix) + pkgpath = os.path.join(self.test_prefix, 'pkgs') + + extra_args = [ + '--experimental', + '--package', + '--package-release=321', + '--package-tool=fpm', + '--package-type=foo', + '--packagepath=%s' % pkgpath, + ] + + self.test_toy_build(extra_args=extra_args) + + toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.321.foo' % EASYBUILD_VERSION) + self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + + def test_package_skip(self): + """Test use of --package with --skip.""" + mock_fpm(self.test_prefix) + pkgpath = os.path.join(self.test_prefix, 'packages') # default path + + self.test_toy_build(['--packagepath=%s' % pkgpath]) + self.assertFalse(os.path.exists(pkgpath), "%s is not created without use of --package" % pkgpath) + + self.test_toy_build(extra_args=['--experimental', '--package', '--skip'], verify=False) + + toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.1.rpm' % EASYBUILD_VERSION) + self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + + def suite(): """ return all the tests in this file """ return TestLoader().loadTestsFromTestCase(ToyBuildTest)