diff --git a/.travis.yml b/.travis.yml index 92e8ab8675..31b7e0a6cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ python: 2.6 env: matrix: # purposely specifying slowest builds first, to gain time overall - - LMOD_VERSION=5.8 - - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - LMOD_VERSION=6.6.3 + - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - LMOD_VERSION=7.7.16 - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl @@ -19,7 +19,7 @@ matrix: include: # also test default configuration with Python 2.7 - python: 2.7 - env: LMOD_VERSION=5.8 + env: LMOD_VERSION=6.6.3 addons: apt: packages: @@ -51,7 +51,7 @@ before_install: # autopep8 1.3.4 is last one to support Python 2.6 - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'autopep8<1.3.5'; else pip install autopep8; fi # optional Python packages for EasyBuild - - pip install GC3Pie pycodestyle python-graph-dot python-hglib PyYAML + - pip install GC3Pie pycodestyle python-graph-dot python-hglib PyYAML requests # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" - git config --global user.email "travis@travis-ci.org" @@ -100,7 +100,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180531.01 21d9704055c4fcf2cd42fe9825dd5925fa508268e4c7c423cb5958a0903d7c2e" + - EB_BOOTSTRAP_EXPECTED="20180916.01 7e7563787a8bab8c30efdbdf95df6ec8ed63230ebcd361fee7b9574cbe9e74ed" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3639edd33b..27987fb9a5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,12 +72,13 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, is_sha256_checksum -from easybuild.tools.filetools import get_source_tarball_from_git, mkdir, move_logs, read_file, remove_file, rmtree2 +from easybuild.tools.filetools import encode_class_name, extract_file, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_logs, read_file, remove_file, rmtree2 from easybuild.tools.filetools import verify_checksum, weld_paths, write_file from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP -from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook +from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP +from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -124,7 +125,7 @@ def extra_options(extra=None): # # INIT # - def __init__(self, ec, hooks=None): + def __init__(self, ec): """ Initialize the EasyBlock instance. :param ec: a parsed easyconfig file (EasyConfig instance) @@ -134,7 +135,7 @@ def __init__(self, ec, hooks=None): self.orig_workdir = os.getcwd() # list of pre- and post-step hooks - self.hooks = hooks or [] + self.hooks = load_hooks(build_option('hooks')) # list of patch/source files, along with checksums self.patches = [] @@ -492,7 +493,7 @@ def fetch_extension_sources(self, skip_checksums=False): 'options': ext_options, } - checksums = ext_options.get('checksums', None) + checksums = ext_options.get('checksums', []) if ext_options.get('source_tmpl', None): fn = resolve_template(ext_options['source_tmpl'], ext_src) @@ -515,14 +516,15 @@ def fetch_extension_sources(self, skip_checksums=False): src_checksum = compute_checksum(src_fn, checksum_type=checksum_type) self.log.info("%s checksum for %s: %s", checksum_type, src_fn, src_checksum) - if checksums: - fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0) - if verify_checksum(src_fn, fn_checksum): - self.log.info('Checksum for extension source %s verified', fn) - elif build_option('ignore_checksums'): - print_warning("Ignoring failing checksum verification for %s" % fn) - else: - raise EasyBuildError('Checksum verification for extension source %s failed', fn) + # verify checksum (if provided) + self.log.debug('Verifying checksums for extension source...') + fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0) + if verify_checksum(src_fn, fn_checksum): + self.log.info('Checksum for extension source %s verified', fn) + elif build_option('ignore_checksums'): + print_warning("Ignoring failing checksum verification for %s" % fn) + else: + raise EasyBuildError('Checksum verification for extension source %s failed', fn) ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) if ext_patches: @@ -537,16 +539,16 @@ def fetch_extension_sources(self, skip_checksums=False): checksum = compute_checksum(patch, checksum_type=checksum_type) self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum) - if checksums: - self.log.debug('Verifying checksums for extension patches...') - for idx, patch in enumerate(ext_patches): - checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx) - if verify_checksum(patch, checksum): - self.log.info('Checksum for extension patch %s verified', patch) - elif build_option('ignore_checksums'): - print_warning("Ignoring failing checksum verification for %s" % patch) - else: - raise EasyBuildError('Checksum for extension patch %s failed', patch) + # verify checksum (if provided) + self.log.debug('Verifying checksums for extension patches...') + for idx, patch in enumerate(ext_patches): + checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx) + if verify_checksum(patch, checksum): + self.log.info('Checksum for extension patch %s verified', patch) + elif build_option('ignore_checksums'): + print_warning("Ignoring failing checksum verification for %s" % patch) + else: + raise EasyBuildError('Checksum for extension patch %s failed', patch) else: self.log.debug('No patches found for extension %s.' % ext_name) @@ -1687,6 +1689,63 @@ def checksum_step(self): else: raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) + def check_checksums_for(self, ent, sub='', source_cnt=None): + """ + Utility method: check whether checksums for all sources/patches are available, for given entity + """ + ec_fn = os.path.basename(self.cfg.path) + checksum_issues = [] + + sources = ent.get('sources', []) + patches = ent.get('patches', []) + checksums = ent.get('checksums', []) + + if source_cnt is None: + source_cnt = len(sources) + patch_cnt, checksum_cnt = len(patches), len(checksums) + + if (source_cnt + patch_cnt) != checksum_cnt: + if sub: + sub = "%s in %s" % (sub, ec_fn) + else: + sub = "in %s" % ec_fn + msg = "Checksums missing for one or more sources/patches %s: " % sub + msg += "found %d sources + %d patches " % (source_cnt, patch_cnt) + msg += "vs %d checksums" % checksum_cnt + checksum_issues.append(msg) + + for fn, checksum in zip(sources + patches, checksums): + if not is_sha256_checksum(checksum): + msg = "Non-SHA256 checksum found for %s: %s" % (fn, checksum) + checksum_issues.append(msg) + + return checksum_issues + + def check_checksums(self): + """ + Check whether a SHA256 checksum is available for all sources & patches (incl. extensions). + + :return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.) + """ + checksum_issues = [] + + # check whether a checksum if available for every source + patch + checksum_issues.extend(self.check_checksums_for(self.cfg)) + + # also check checksums for extensions + for ext in self.cfg['exts_list']: + # just skip extensions for which only a name is specified + # those are just there to check for things that are in the "standard library" + if not isinstance(ext, basestring): + ext_name = ext[0] + # take into account that extension may be a 2-tuple with just name/version + ext_opts = ext[2] if len(ext) == 3 else {} + # only a single source per extension is supported (see source_tmpl) + res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=1) + checksum_issues.extend(res) + + return checksum_issues + def extract_step(self): """ Unpack the source files. @@ -2346,6 +2405,17 @@ def cleanup_step(self): self.restore_iterate_opts() + def invalidate_module_caches(self, modpath): + """Helper method to invalidate module caches for specified module path.""" + # invalidate relevant 'module avail'/'module show' cache entries + # consider both paths: for short module name, and subdir indicated by long module name + paths = [modpath] + if self.mod_subdir: + paths.append(os.path.join(modpath, self.mod_subdir)) + + for path in paths: + invalidate_module_caches_for(path) + def make_module_step(self, fake=False): """ Generate module file @@ -2395,14 +2465,7 @@ def make_module_step(self, fake=False): self.log.info(diff_msg) print_msg(diff_msg, log=self.log) - # invalidate relevant 'module avail'/'module show' cache entries - # consider both paths: for short module name, and subdir indicated by long module name - paths = [modpath] - if self.mod_subdir: - paths.append(os.path.join(modpath, self.mod_subdir)) - - for path in paths: - invalidate_module_caches_for(path) + self.invalidate_module_caches(modpath) # only update after generating final module file if not fake: @@ -2507,8 +2570,11 @@ def _skip_step(self, step, skippable): force = build_option('force') or build_option('rebuild') skip = False - # skip step if specified as individual (skippable) step - if skippable and (self.skip or step in self.cfg['skipsteps']): + # under --skip, sanity check is not skipped + cli_skip = self.skip and step != SANITYCHECK_STEP + + # skip step if specified as individual (skippable) step, or if --skip is used + if skippable and (cli_skip or step in self.cfg['skipsteps']): self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) skip = True @@ -2654,7 +2720,7 @@ def install_step_spec(initial): steps_part3 = [ (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False), (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step], True), - (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], False), + (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], True), (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step], False), (MODULE_STEP, 'creating module', [lambda x: x.make_module_step], False), (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step], False), @@ -2717,7 +2783,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, hooks=None): +def build_and_install_one(ecdict, init_env): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -2755,7 +2821,7 @@ def build_and_install_one(ecdict, init_env, hooks=None): try: app_class = get_easyblock_class(easyblock, name=name) - app = app_class(ecdict['ec'], hooks=hooks) + app = app_class(ecdict['ec']) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), @@ -2816,13 +2882,14 @@ def build_and_install_one(ecdict, init_env, hooks=None): buildstats = get_build_stats(app, start_time, build_option('command_line')) _log.info("Build stats: %s" % buildstats) - if build_option("minimal_toolchains"): - # for reproducability we dump out the parsed easyconfig since the contents are affected when - # --minimal-toolchains (and --use-existing-modules) is used - # TODO --try-toolchain needs to be fixed so this doesn't play havoc with it's usability - reprod_spec = os.path.join(new_log_dir, 'reprod', ec_filename) + # for reproducability we dump out the fully processed easyconfig since the contents can be affected + # by subtoolchain resolution (and related options) and/or hooks + reprod_spec = os.path.join(new_log_dir, 'reprod', ec_filename) + try: app.cfg.dump(reprod_spec) - _log.debug("Dumped easyconfig tweaked via --minimal-toolchains to %s", reprod_spec) + _log.info("Dumped fully processed easyconfig to %s", reprod_spec) + except NotImplementedError as err: + _log.warn("Unable to dumped fully processed easyconfig to %s: %s", reprod_spec, err) try: # upload easyconfig (and patch files) to central repository diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index d090b33bd6..fc02703501 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -90,7 +90,8 @@ 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], - 'github_account': [None, "GitHub account name to be used to resolve template values in source URLs", BUILD], + 'github_account': ['%(namelower)s', "GitHub account name to be used to resolve template values in source URLs", + BUILD], 'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], 'maxparallel': [None, 'Max degree of parallelism', BUILD], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index fa51ce812b..a447b058f8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -34,6 +34,7 @@ :author: Toon Willems (Ghent University) :author: Ward Poelmans (Ghent University) :author: Alan O'Cais (Juelich Supercomputing Centre) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) """ import copy @@ -55,11 +56,11 @@ from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict -from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_naming_scheme from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name, read_file, write_file +from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name @@ -134,6 +135,39 @@ def cache_aware_func(toolchain): return cache_aware_func +def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands): + """ + Returns unique version for subtoolchain, in tc dict. + If there is no unique version: + * use '' for dummy, if dummy is not skipped. + * return None for skipped subtoolchains, that is, + optional toolchains or dummy without add_dummy_to_minimal_toolchains. + * in all other cases, raises an exception. + """ + current_tc_name, current_tc_version = current_tc['name'], current_tc['version'] + + uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) + # init with "skipped" + subtoolchain_version = None + + # dummy toolchain: bottom of the hierarchy + if subtoolchain_name == DUMMY_TOOLCHAIN_NAME: + if build_option('add_dummy_to_minimal_toolchains'): + subtoolchain_version = '' + elif len(uniq_subtc_versions) == 1: + subtoolchain_version = list(uniq_subtc_versions)[0] + elif len(uniq_subtc_versions) == 0: + if subtoolchain_name not in optional_toolchains: + # raise error if the subtoolchain considered now is not optional + raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", + subtoolchain_name, current_tc_name) + else: + raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", + subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions))) + + return subtoolchain_version + + @toolchain_hierarchy_cache def get_toolchain_hierarchy(parent_toolchain): """ @@ -143,19 +177,45 @@ def get_toolchain_hierarchy(parent_toolchain): The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains build option is enabled. + The most complex hierarchy we have now is goolfc which works as follows: + + goolfc + / \ + gompic golfc(*) + \ / \ (*) optional toolchains, not compulsory for backwards compatibility + gcccuda golf(*) + \ / + GCC + / | + GCCcore(*) | + \ | + (dummy: only considered if --add-dummy-to-minimal-toolchains configuration option is enabled) + :param parent_toolchain: dictionary with name/version of parent toolchain """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain('') subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) - - current_tc_name, current_tc_version = parent_toolchain['name'], parent_toolchain['version'] - subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None + optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) + composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) # the parent toolchain is at the top of the hierarchy toolchain_hierarchy = [parent_toolchain] - - while subtoolchain_name: + # use a queue to handle a breadth-first-search of the hierarchy, + # which is required to take into account the potential for multiple subtoolchains + bfs_queue = [parent_toolchain] + visited = set() + + while bfs_queue: + current_tc = bfs_queue.pop() + current_tc_name, current_tc_version = current_tc['name'], current_tc['version'] + subtoolchain_names = subtoolchains[current_tc_name] + # if current toolchain has no subtoolchains, consider next toolchain in queue + if subtoolchain_names is None: + continue + # make sure we always have a list of subtoolchains, even if there's only one + if not isinstance(subtoolchain_names, list): + subtoolchain_names = [subtoolchain_names] # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain path = robot_find_easyconfig(current_tc_name, current_tc_version) if path is None: @@ -195,44 +255,26 @@ def get_toolchain_hierarchy(parent_toolchain): cands.append({'name': depdep['name'], 'version': depdep['version'] + depdep['versionsuffix']}) cands.append(depdep['toolchain']) - # only retain candidates that match subtoolchain name - cands = [c for c in cands if c['name'] == subtoolchain_name] - - uniq_subtc_versions = set([subtc['version'] for subtc in cands]) - - if len(uniq_subtc_versions) == 1: - subtoolchain_version = cands[0]['version'] - - elif len(uniq_subtc_versions) == 0: - # only retain GCCcore as subtoolchain if version was found - if subtoolchain_name == GCCcore.NAME: - _log.info("No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.", - subtoolchain_name) - subtoolchain_name = GCCcore.SUBTOOLCHAIN - subtoolchain_version = '' - # dummy toolchain: end of the line - elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME: - subtoolchain_version = '' - else: - raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", - subtoolchain_name, current_tc_name) - else: - if subtoolchain_name == DUMMY_TOOLCHAIN_NAME: - # Don't care about multiple versions of dummy - _log.info("Ignoring multiple versions of %s in toolchain hierarchy", DUMMY_TOOLCHAIN_NAME) - subtoolchain_version = '' - else: - raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", - subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions))) - - if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'): - # we're done - break - - # add to hierarchy and move to next - current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version - subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None - toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version}) + for dep in subtoolchain_names: + # try to find subtoolchains with the same version as the parent + # only do this for composite toolchains, not single-compiler toolchains, whose + # versions match those of the component instead of being e.g. "2018a". + if dep in composite_toolchains: + ecfile = robot_find_easyconfig(dep, current_tc_version) + if ecfile is not None: + cands.append({'name': dep, 'version': current_tc_version}) + + # only retain candidates that match subtoolchain names + cands = [c for c in cands if c['name'] in subtoolchain_names] + + for subtoolchain_name in subtoolchain_names: + subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands) + # add to hierarchy and move to next + if subtoolchain_version is not None and subtoolchain_name not in visited: + tc = {'name': subtoolchain_name, 'version': subtoolchain_version} + toolchain_hierarchy.insert(0, tc) + bfs_queue.insert(0, tc) + visited.add(subtoolchain_name) _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy @@ -434,22 +476,38 @@ def parse(self): # we need toolchain to be set when we call _parse_dependency for key in ['toolchain'] + local_vars.keys(): # validations are skipped, just set in the config - # do not store variables we don't need if key in self._config.keys(): - if key in ['dependencies']: - self[key] = [self._parse_dependency(dep) for dep in local_vars[key]] - elif key in ['builddependencies']: - self[key] = [self._parse_dependency(dep, build_only=True) for dep in local_vars[key]] - elif key in ['hiddendependencies']: - self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]] - else: - self[key] = local_vars[key] + self[key] = local_vars[key] self.log.info("setting config option %s: value %s (type: %s)", key, self[key], type(self[key])) elif key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') + # do not store variables we don't need else: - self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key])) + self.log.debug("Ignoring unknown easyconfig parameter %s (value: %s)" % (key, local_vars[key])) + + # trigger parse hook + # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters + # (see also comment in resolve_template) + hooks = load_hooks(build_option('hooks')) + prev_enable_templating = self.enable_templating + self.enable_templating = False + + parse_hook_msg = None + if self.path: + parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path)) + + run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) + + # parse dependency specifications + # it's important that templating is still disabled at this stage! + self.log.info("Parsing dependency specifications...") + self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] + self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] + self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] + + # restore templating + self.enable_templating = prev_enable_templating # update templating dictionary self.generate_template_values() @@ -701,9 +759,15 @@ def dump(self, fp): self.generate_template_values() templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) - # reverse map of templates longer than 2 characters, to inject template values where possible, sorted on length - keys = sorted(self.template_values, key=lambda k: len(self.template_values[k]), reverse=True) - templ_val = OrderedDict([(self.template_values[k], k) for k in keys if len(self.template_values[k]) > 2]) + # create reverse map of templates, to inject template values where possible + # longer template values are considered first, shorter template keys get preference over longer ones + sorted_keys = sorted(self.template_values, key=lambda k: (len(self.template_values[k]), -len(k)), reverse=True) + templ_val = OrderedDict([]) + for key in sorted_keys: + # shortest template 'key' is retained in case of duplicates ('namelower' is preferred over 'github_account') + # only template values longer than 2 characters are retained + if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2: + templ_val[self.template_values[key]] = key ectxt = self.parser.dump(self, default_values, templ_const, templ_val) self.log.debug("Dumped easyconfig: %s", ectxt) diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index e485dd260b..824546a1d5 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -32,7 +32,8 @@ import sys from vsc.utils import fancylogger -from easybuild.tools.build_log import print_msg +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.utilities import only_if_module_is_available try: @@ -144,16 +145,25 @@ def check_easyconfigs_style(easyconfigs, verbose=False): return result.total_errors -def cmdline_easyconfigs_style_check(paths): +def cmdline_easyconfigs_style_check(ecs): """ - Run easyconfigs style check of each of the specified paths, triggered from 'eb' command line + Run easyconfigs style check of each of the specified easyconfigs, triggered from 'eb' command line - :param paths: list of paths to easyconfig files to check + :param ecs: list of easyconfigs to check, could be either file paths or EasyConfig instances :return: True when style check passed on all easyconfig files, False otherwise """ - print_msg("Running style check on %d easyconfig(s)..." % len(paths), prefix=False) + print_msg("\nRunning style check on %d easyconfig(s)...\n" % len(ecs), prefix=False) style_check_passed = True - for path in paths: + for ec in ecs: + # if an EasyConfig instance is provided, just grab the corresponding file path + if isinstance(ec, EasyConfig): + path = ec.path + elif isinstance(ec, basestring): + path = ec + else: + raise EasyBuildError("Value of unknown type encountered in cmdline_easyconfigs_style_check: %s (type: %s)", + ec, type(ec)) + if check_easyconfigs_style([path]) == 0: res = 'PASS' else: diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fdfe0272ea..963129f743 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -91,9 +91,9 @@ ('FTPGNOME_SOURCE', 'http://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', 'http download for gnome ftp server'), ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive', - 'GitHub source URL (requires github_account easyconfig parameter to be specified)'), + 'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'), ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive', - 'GitHub source URL (lowercase name, requires github_account easyconfig parameter to be specified)'), + 'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'), ('GNU_SAVANNAH_SOURCE', 'http://download-mirror.savannah.gnu.org/releases/%(namelower)s', 'download.savannah.gnu.org source url'), ('GNU_SOURCE', 'http://ftpmirror.gnu.org/gnu/%(namelower)s', diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c0460c3c71..f4683711af 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -49,6 +49,7 @@ from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars +from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env @@ -368,6 +369,7 @@ def parse_easyconfigs(paths, validate=True): """ easyconfigs = [] generated_ecs = False + for (path, generated) in paths: path = os.path.abspath(path) # keep track of whether any files were generated @@ -377,12 +379,13 @@ def parse_easyconfigs(paths, validate=True): try: ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) for ec_file in ec_files: - # only pass build specs when not generating easyconfig files kwargs = {'validate': validate} + # only pass build specs when not generating easyconfig files if not build_option('try_to_generate'): kwargs['build_specs'] = build_option('build_specs') - ecs = process_easyconfig(ec_file, **kwargs) - easyconfigs.extend(ecs) + + easyconfigs.extend(process_easyconfig(ec_file, **kwargs)) + except IOError, err: raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err) @@ -594,3 +597,59 @@ def categorize_files_by_type(paths): res['easyconfigs'].append(path) return res + + +def check_sha256_checksums(ecs, whitelist=None): + """ + Check whether all provided (parsed) easyconfigs have SHA256 checksums for sources & patches. + + :param whitelist: list of regex patterns on easyconfig filenames; check is skipped for matching easyconfigs + :return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.) + """ + checksum_issues = [] + + if whitelist is None: + whitelist = [] + + for ec in ecs: + # skip whitelisted software + ec_fn = os.path.basename(ec.path) + if any(re.match(regex, ec_fn) for regex in whitelist): + _log.info("Skipping SHA256 checksum check for %s because of whitelist (%s)", ec.path, whitelist) + continue + + eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) + checksum_issues.extend(eb_class(ec).check_checksums()) + + return checksum_issues + + +def run_contrib_checks(ecs): + """Run contribution check on specified easyconfigs.""" + + def print_result(checks_passed, label): + """Helper function to print result of last group of checks.""" + if checks_passed: + print_msg("\n>> All %s checks PASSed!" % label, prefix=False) + else: + print_msg("\n>> One or more %s checks FAILED!" % label, prefix=False) + + # start by running style checks + style_check_ok = cmdline_easyconfigs_style_check(ecs) + print_result(style_check_ok, "style") + + # check whether SHA256 checksums are in place + print_msg("\nChecking for SHA256 checksums in %d easyconfig(s)...\n" % len(ecs), prefix=False) + sha256_checksums_ok = True + for ec in ecs: + sha256_checksum_fails = check_sha256_checksums([ec]) + if sha256_checksum_fails: + sha256_checksums_ok = False + msgs = ['[FAIL] %s' % ec.path] + sha256_checksum_fails + else: + msgs = ['[PASS] %s' % ec.path] + print_msg('\n'.join(msgs), prefix=False) + + print_result(sha256_checksums_ok, "SHA256 checksums") + + return style_check_ok and sha256_checksums_ok diff --git a/easybuild/main.py b/easybuild/main.py index d0873b3596..38444a5ea5 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available +from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize @@ -106,27 +106,24 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure - :param hooks: list of defined pre- and post-step hooks """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) - run_hook(START, hooks) - res = [] for ec in ecs: ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -165,11 +162,31 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, ho res.append((ec, ec_res)) - run_hook(END, hooks) - return res +def run_contrib_style_checks(ecs, check_contrib, check_style): + """ + Handle running of contribution and style checks on specified easyconfigs (if desired). + + :return: boolean indicating whether or not any checks were actually performed + """ + check_actions = { + 'contribution': (check_contrib, run_contrib_checks), + 'style': (check_style, cmdline_easyconfigs_style_check), + } + for check_label, (run_check, check_function) in sorted(check_actions.items()): + if run_check: + _log.info("Running %s checks on %d specified easyconfigs...", check_label, len(ecs)) + if check_function(ecs): + print_msg("\n>> All %s checks PASSed!\n" % check_label, prefix=False) + else: + print_msg('', prefix=False) + raise EasyBuildError("One or more %s checks FAILED!" % check_label) + + return check_contrib or check_style + + def check_root_usage(allow_use_as_root=False): """ Check whether we are running as root, and act accordingly @@ -187,6 +204,12 @@ def check_root_usage(allow_use_as_root=False): "so let's end this here.") +def clean_exit(logfile, tmpdir, testing, silent=False): + """Small utility function to perform a clean exit.""" + cleanup(logfile, tmpdir, testing, silent=silent) + sys.exit(0) + + def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. @@ -256,6 +279,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) + # load hook implementations (if any) + hooks = load_hooks(options.hooks) + + run_hook(START, hooks) + if modtool is None: modtool = modules_tool(testing=testing) @@ -310,8 +338,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): search_query, ] if any(early_stop_options): - cleanup(logfile, eb_tmpdir, testing, silent=True) - sys.exit(0) + clean_exit(logfile, eb_tmpdir, testing, silent=True) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) @@ -365,18 +392,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 - if options.check_style: - _log.debug("Running style check...") - if cmdline_easyconfigs_style_check([path[0] for path in paths]): - print_msg("All style checks passed!", prefix=False) - cleanup(logfile, eb_tmpdir, testing) - sys.exit(0) - else: - raise EasyBuildError("One or more style checks FAILED!") - # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) + # handle --check-contrib & --check-style options + if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): + clean_exit(logfile, eb_tmpdir, testing) + # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") @@ -392,8 +414,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) - cleanup(logfile, eb_tmpdir, testing) - sys.exit(0) + clean_exit(logfile, eb_tmpdir, testing) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short @@ -453,31 +474,26 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] if any(no_ec_opts) or any(stop_options): - cleanup(logfile, eb_tmpdir, testing) - sys.exit(0) + clean_exit(logfile, eb_tmpdir, testing) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) - cleanup(logfile, eb_tmpdir, testing, silent=True) - sys.exit(0) + clean_exit(logfile, eb_tmpdir, testing, silent=True) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") - cleanup(logfile, eb_tmpdir, testing) - sys.exit(0) + clean_exit(logfile, eb_tmpdir, testing) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - hooks = load_hooks(options.hooks) - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, - exit_on_failure=exit_on_failure, hooks=hooks) + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] @@ -500,6 +516,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) + run_hook(END, hooks) + # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 4b897f0be6..8d497ef6d5 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -54,7 +54,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180531.01' +EB_BOOTSTRAP_VERSION = '20180916.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -87,6 +87,7 @@ EASYBUILD_BOOTSTRAP_SOURCEPATH = os.environ.pop('EASYBUILD_BOOTSTRAP_SOURCEPATH', None) EASYBUILD_BOOTSTRAP_SKIP_STAGE0 = os.environ.pop('EASYBUILD_BOOTSTRAP_SKIP_STAGE0', False) +EASYBUILD_BOOTSTRAP_FORCE_VERSION = os.environ.pop('EASYBUILD_BOOTSTRAP_FORCE_VERSION', None) # keep track of original environment (after clearing PYTHONPATH) orig_os_environ = copy.deepcopy(os.environ) @@ -474,7 +475,7 @@ def stage0(tmpdir): return distribute_egg_dir -def stage1(tmpdir, sourcepath, distribute_egg_dir): +def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion): """STAGE 1: temporary install EasyBuild using distribute's easy_install.""" print('\n') @@ -523,7 +524,10 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir): cmd.append(source_tarballs[VSC_BASE]) else: # install meta-package easybuild from PyPI - cmd.append('easybuild') + if forcedversion: + cmd.append('easybuild==%s' % forcedversion) + else: + cmd.append('easybuild') # install vsc-base again at the end, to avoid that the one available on the system is used instead post_vsc_base = cmd[:] @@ -823,6 +827,10 @@ def main(): if sourcepath is not None: info("Fetching sources from %s..." % sourcepath) + forcedversion = EASYBUILD_BOOTSTRAP_FORCE_VERSION + if forcedversion: + info("Forcing specified version %s..." % forcedversion) + # create temporary dir for temporary installations tmpdir = tempfile.mkdtemp() debug("Going to use %s as temporary directory" % tmpdir) @@ -869,7 +877,7 @@ def main(): distribute_egg_dir = stage0(tmpdir) # STAGE 1: install EasyBuild using easy_install to tmp dir - templates = stage1(tmpdir, sourcepath, distribute_egg_dir) + templates = stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion) # add location to easy_install provided through stage0 to $PATH # this must be done *after* stage1, since $PATH is reset during stage1 diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index 8896dd379a..a6de5f0a20 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -29,12 +29,13 @@ """ from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK + class Foss(Gompi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'foss' - SUBTOOLCHAIN = Gompi.NAME - + SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME] diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py index baa5b3e9f0..742ee0239f 100644 --- a/easybuild/toolchains/fosscuda.py +++ b/easybuild/toolchains/fosscuda.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.gompic import Gompic +from easybuild.toolchains.golfc import Golfc from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -37,4 +38,4 @@ class Fosscuda(Gompic, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC+CUDA, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'fosscuda' - SUBTOOLCHAIN = Gompic.NAME + SUBTOOLCHAIN = [Gompic.NAME, Golfc.NAME] diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index 26f07ea028..fd234ecbbd 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -29,9 +29,11 @@ """ from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME class GccToolchain(GCCcore): """Simple toolchain with just the GCC compilers.""" NAME = 'GCC' COMPILER_MODULE_NAME = [NAME] - SUBTOOLCHAIN = GCCcore.NAME + SUBTOOLCHAIN = [GCCcore.NAME, DUMMY_TOOLCHAIN_NAME] + OPTIONAL = False diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index a61a152f3d..6c06f85d3b 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -37,3 +37,6 @@ class GCCcore(Gcc): # Replace the default compiler module name with our own COMPILER_MODULE_NAME = [NAME] SUBTOOLCHAIN = DUMMY_TOOLCHAIN_NAME + # GCCcore is only guaranteed to be present in recent toolchains + # for old versions of some toolchains (GCC, intel) it is not part of the hierarchy and hence optional + OPTIONAL = True diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index ead8f081e7..818809f37c 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.gimpi import Gimpi +from easybuild.toolchains.gmkl import Gmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -38,4 +39,4 @@ class Gimkl(Gimpi, IntelMKL, IntelFFTW): """Compiler toolchain with GCC, Intel MPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers.""" NAME = 'gimkl' - SUBTOOLCHAIN = Gimpi.NAME + SUBTOOLCHAIN = [Gimpi.NAME, Gmkl.NAME] diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 1350617128..7593b43fbe 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.gimpi import Gimpi +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -37,4 +38,4 @@ class Giolf(Gimpi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, IntelMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'giolf' - SUBTOOLCHAIN = Gimpi.NAME + SUBTOOLCHAIN = [Gimpi.NAME, Golf.NAME] diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index 654eebfe55..439ad00780 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -30,6 +30,7 @@ """ from easybuild.toolchains.gimpic import Gimpic +from easybuild.toolchains.golfc import Golfc from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -37,5 +38,5 @@ class Giolfc(Gimpic, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC+CUDA, IntelMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'giolfc' - SUBTOOLCHAIN = Gimpic.NAME + SUBTOOLCHAIN = [Gimpic.NAME, Golfc.NAME] diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py index ca169c9b25..6eb80729d2 100644 --- a/easybuild/toolchains/gmkl.py +++ b/easybuild/toolchains/gmkl.py @@ -43,3 +43,4 @@ class Gmkl(GccToolchain, IntelMKL, IntelFFTW): """ NAME = 'gmkl' SUBTOOLCHAIN = GccToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index a5b9a77cec..92eacb2b88 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -33,14 +33,16 @@ """ from easybuild.toolchains.gcccuda import GccCUDA +from easybuild.toolchains.gmkl import Gmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Gmklc(GccCUDA, IntelMKL, IntelFFTW): +class Gmklc(GccCUDA, Gmkl, IntelMKL, IntelFFTW): """ Compiler toolchain with GCC, Intel Math Kernel Library (MKL) and Intel FFTW wrappers and CUDA. """ NAME = 'gmklc' - SUBTOOLCHAIN = GccCUDA.NAME + SUBTOOLCHAIN = [GccCUDA.NAME, Gmkl.NAME] + OPTIONAL = True diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 949a0f9b92..6028abbc4e 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -33,6 +33,7 @@ """ from easybuild.toolchains.gmpich import Gmpich +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -41,4 +42,4 @@ class Gmpolf(Gmpich, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpolf' - SUBTOOLCHAIN = Gmpich.NAME + SUBTOOLCHAIN = [Gmpich.NAME, Golf.NAME] diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 4ec094689a..7f6aa6ef1e 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -32,6 +32,7 @@ """ from easybuild.toolchains.gmvapich2 import Gmvapich2 +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -40,4 +41,4 @@ class Gmvolf(Gmvapich2, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, MVAPICH2, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmvolf' - SUBTOOLCHAIN = Gmvapich2.NAME + SUBTOOLCHAIN = [Gmvapich2.NAME, Golf.NAME] diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py new file mode 100644 index 0000000000..186666c6e1 --- /dev/null +++ b/easybuild/toolchains/goblf.py @@ -0,0 +1,42 @@ +## +# Copyright 2013-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for foss compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.linalg.blis import Blis +from easybuild.toolchains.linalg.lapack import Lapack +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Goblf(Gompi, Blis, Lapack, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW.""" + NAME = 'goblf' + SUBTOOLCHAIN = Gompi.NAME diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py index 32ce7ecd42..e2257aa322 100644 --- a/easybuild/toolchains/golf.py +++ b/easybuild/toolchains/golf.py @@ -38,3 +38,4 @@ class Golf(GccToolchain, OpenBLAS, Fftw): """Compiler toolchain with GCC, OpenBLAS, and FFTW.""" NAME = 'golf' SUBTOOLCHAIN = GccToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py new file mode 100644 index 0000000000..ed88ccfa31 --- /dev/null +++ b/easybuild/toolchains/golfc.py @@ -0,0 +1,42 @@ +## +# Copyright 2013-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for golfc compiler toolchain (includes GCC+CUDA, OpenBLAS, LAPACK, and FFTW). + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.gcccuda import GccCUDA +from easybuild.toolchains.golf import Golf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS + + +class Golfc(GccCUDA, Golf, OpenBLAS, Fftw): + """Compiler toolchain with GCC+CUDA, OpenBLAS, and FFTW.""" + NAME = 'golfc' + SUBTOOLCHAIN = [GccCUDA.NAME, Golf.NAME] + OPTIONAL = True diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py index 04b2dbbab1..733a49c407 100644 --- a/easybuild/toolchains/gomkl.py +++ b/easybuild/toolchains/gomkl.py @@ -32,6 +32,7 @@ """ from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.gmkl import Gmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -39,4 +40,4 @@ class Gomkl(Gompi, IntelMKL, IntelFFTW): """Compiler toolchain with GCC, OpenMPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers.""" NAME = 'gomkl' - SUBTOOLCHAIN = Gompi.NAME + SUBTOOLCHAIN = [Gompi.NAME, Gmkl.NAME] diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index d3ca69fc0b..8b428cc827 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -33,6 +33,7 @@ """ from easybuild.toolchains.gompic import Gompic +from easybuild.toolchains.gmklc import Gmklc from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -40,4 +41,4 @@ class Gomklc(Gompic, IntelMKL, IntelFFTW): """Compiler toolchain with GCC, Open MPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers and Cuda.""" NAME = 'gomklc' - SUBTOOLCHAIN = Gompic.NAME + SUBTOOLCHAIN = [Gompic.NAME, Gmklc.NAME] diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 7ddc1cc9c6..363b43754c 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -37,4 +38,4 @@ class Goolf(Gompi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'goolf' - SUBTOOLCHAIN = Gompi.NAME + SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME] diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index e54d4353a9..1783f08cff 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.gompic import Gompic +from easybuild.toolchains.golfc import Golfc from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -36,5 +37,4 @@ class Goolfc(Gompic, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC+CUDA, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'goolfc' - SUBTOOLCHAIN = Gompic.NAME - + SUBTOOLCHAIN = [Gompic.NAME, Golfc.NAME] diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index c53d79274a..0162ddfc51 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.gpsmpi import Gpsmpi +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -39,4 +40,4 @@ class Gpsolf(Gpsmpi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, Parastation MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gpsolf' - SUBTOOLCHAIN = Gpsmpi.NAME + SUBTOOLCHAIN = [Gpsmpi.NAME, Golf.NAME] diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index e6c0ea3372..efa6852e87 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -30,6 +30,7 @@ """ from easybuild.toolchains.gsmpi import Gsmpi +from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK @@ -37,4 +38,4 @@ class Gsolf(Gsmpi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, SpectrumMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gsolf' - SUBTOOLCHAIN = Gsmpi.NAME + SUBTOOLCHAIN = [Gsmpi.NAME, Golf.NAME] diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index a9f9441f3b..1c9aa2e7aa 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -32,10 +32,12 @@ from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort # Need to import the GCCcore class so I can get the name from there from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME class IccIfort(IntelIccIfort): """Compiler toolchain with Intel compilers (icc/ifort).""" NAME = 'iccifort' # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, # in particular in a hierarchical module naming scheme - SUBTOOLCHAIN = GCCcore.NAME + SUBTOOLCHAIN = [GCCcore.NAME, DUMMY_TOOLCHAIN_NAME] + OPTIONAL = False diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 2aa6c32dd9..2bccff4ec9 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.iimpi import Iimpi +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -41,4 +42,4 @@ class Ictce(Iimpi, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'ictce' - SUBTOOLCHAIN = Iimpi.NAME + SUBTOOLCHAIN = [Iimpi.NAME, Iimkl.NAME] diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 0738421f16..afb5e17c0f 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -43,3 +43,4 @@ class Iimkl(IccIfort, IntelMKL, IntelFFTW): """ NAME = 'iimkl' SUBTOOLCHAIN = IccIfort.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index 5bb2cd294b..0a8e1ddf8a 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -33,14 +33,16 @@ """ from easybuild.toolchains.iccifortcuda import IccIfortCUDA +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Iimklc(IccIfortCUDA, IntelMKL, IntelFFTW): +class Iimklc(IccIfortCUDA, Iimkl, IntelMKL, IntelFFTW): """ Compiler toolchain with Intel compilers (icc/ifort), CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'iimklc' - SUBTOOLCHAIN = IccIfortCUDA.NAME + SUBTOOLCHAIN = [IccIfortCUDA.NAME, Iimkl.NAME] + OPTIONAL = True diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 37e8c0815b..4dc79c0f63 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -30,6 +30,7 @@ """ from easybuild.toolchains.impich import Impich +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -39,4 +40,4 @@ class Impmkl(Impich, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'impmkl' - SUBTOOLCHAIN = Impich.NAME + SUBTOOLCHAIN = [Impich.NAME, Iimkl.NAME] diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index 7888bce085..0bcb53be41 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.ipsmpi import Ipsmpi +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -39,4 +40,4 @@ class IntelPara(Ipsmpi, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'intel-para' - SUBTOOLCHAIN = Ipsmpi.NAME + SUBTOOLCHAIN = [Ipsmpi.NAME, Iimkl] diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index 04fb90f47b..4a23a3c7d9 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.iimpi import Iimpi +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -41,4 +42,4 @@ class Intel(Iimpi, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'intel' - SUBTOOLCHAIN = Iimpi.NAME + SUBTOOLCHAIN = [Iimpi.NAME, Iimkl.NAME] diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index 8d42363920..0211714f88 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -29,6 +29,7 @@ """ from easybuild.toolchains.iimpic import Iimpic +from easybuild.toolchains.iimklc import Iimklc from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -39,4 +40,4 @@ class Intelcuda(Iimpic, IntelMKL, IntelFFTW): NAME = 'intelcuda' - SUBTOOLCHAIN = Iimpic.NAME + SUBTOOLCHAIN = [Iimpic.NAME, Iimklc.NAME] diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index 1f7d1a42aa..7d4b1d4127 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.iompi import Iompi +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -41,4 +42,4 @@ class Iomkl(Iompi, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'iomkl' - SUBTOOLCHAIN = Iompi.NAME + SUBTOOLCHAIN = [Iompi.NAME, Iimkl.NAME] diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py index fc4e9e7a40..a7fab2a30a 100644 --- a/easybuild/toolchains/iomklc.py +++ b/easybuild/toolchains/iomklc.py @@ -32,6 +32,7 @@ """ from easybuild.toolchains.iompic import Iompic +from easybuild.toolchains.iimklc import Iimklc from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -42,4 +43,4 @@ class Iomklc(Iompic, IntelMKL, IntelFFTW): CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'iomklc' - SUBTOOLCHAIN = Iompic.NAME + SUBTOOLCHAIN = [Iompic.NAME, Iimklc.NAME] diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index e140e82b5b..1177c2235d 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -31,6 +31,7 @@ """ from easybuild.toolchains.iccifort import IccIfort +from easybuild.toolchains.iimkl import Iimkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.mpi.mpich2 import Mpich2 from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -42,4 +43,4 @@ class Ismkl(IccIfort, Mpich2, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'ismkl' - SUBTOOLCHAIN = IccIfort.NAME + SUBTOOLCHAIN = [IccIfort.NAME, Iimkl.NAME] diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py new file mode 100644 index 0000000000..d0711413f0 --- /dev/null +++ b/easybuild/toolchains/linalg/blis.py @@ -0,0 +1,44 @@ +## +# Copyright 2013-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for BLIS as toolchain linear algebra library. + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.tools.toolchain.linalg import LinAlg + + +TC_CONSTANT_BLIS = 'BLIS' + + +class Blis(LinAlg): + """ + Trivial class, provides BLIS support. + """ + BLAS_MODULE_NAME = ['BLIS'] + BLAS_LIB = ['blis'] + BLAS_FAMILY = TC_CONSTANT_BLIS diff --git a/easybuild/toolchains/pgi.py b/easybuild/toolchains/pgi.py index 7ed5f8448a..f05f83c75b 100644 --- a/easybuild/toolchains/pgi.py +++ b/easybuild/toolchains/pgi.py @@ -33,6 +33,7 @@ from easybuild.toolchains.compiler.pgi import Pgi from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME class PgiToolchain(Pgi): @@ -40,4 +41,5 @@ class PgiToolchain(Pgi): NAME = 'PGI' # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, # in particular in a hierarchical module naming scheme - SUBTOOLCHAIN = GCCcore.NAME + SUBTOOLCHAIN = [GCCcore.NAME, DUMMY_TOOLCHAIN_NAME] + OPTIONAL = False diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py index 2aeac0290f..f6083eb139 100644 --- a/easybuild/toolchains/pmkl.py +++ b/easybuild/toolchains/pmkl.py @@ -43,3 +43,4 @@ class Pmkl(PgiToolchain, IntelMKL, IntelFFTW): """ NAME = 'pmkl' SUBTOOLCHAIN = PgiToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index e71c17d1a3..ba37355221 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -32,6 +32,7 @@ """ from easybuild.toolchains.pompi import Pompi +from easybuild.toolchains.pmkl import Pmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL @@ -42,4 +43,4 @@ class Pomkl(Pompi, IntelMKL, IntelFFTW): Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'pomkl' - SUBTOOLCHAIN = Pompi.NAME + SUBTOOLCHAIN = [Pompi.NAME, Pmkl.NAME] diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 5180714438..d9b6e22eaa 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -142,6 +142,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'github_user', 'github_org', 'group', + 'hooks', 'ignore_dirs', 'job_backend_config', 'job_cores', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ba8cc0c5ff..b2fb9186d0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,11 @@ from easybuild.tools.config import build_option from easybuild.tools import run +try: + import requests + HAVE_REQUESTS = True +except ImportError: + HAVE_REQUESTS = False _log = fancylogger.getLogger('filetools', fname=False) @@ -495,26 +500,47 @@ def download_file(filename, url, path, forced=False): attempt_cnt = 0 # use custom HTTP header - url_req = urllib2.Request(url, headers={'User-Agent': 'EasyBuild', "Accept" : "*/*"}) + headers = {'User-Agent': 'EasyBuild', 'Accept': '*/*'} + # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' + url_req = urllib2.Request(url, headers=headers) + used_urllib = urllib2 while not downloaded and attempt_cnt < max_attempts: try: - # urllib2 does the right thing for http proxy setups, urllib does not! - url_fd = urllib2.urlopen(url_req, timeout=timeout) - _log.debug('response code for given url %s: %s' % (url, url_fd.getcode())) + if used_urllib is urllib2: + # urllib2 does the right thing for http proxy setups, urllib does not! + url_fd = urllib2.urlopen(url_req, timeout=timeout) + status_code = url_fd.getcode() + else: + response = requests.get(url, headers=headers, stream=True, timeout=timeout) + status_code = response.status_code + response.raise_for_status() + url_fd = response.raw + url_fd.decode_content = True + _log.debug('response code for given url %s: %s' % (url, status_code)) write_file(path, url_fd.read(), forced=forced, backup=True) _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True url_fd.close() - except urllib2.HTTPError as err: - if 400 <= err.code <= 499: - _log.warning("URL %s was not found (HTTP response code %s), not trying again" % (url, err.code)) + except used_urllib.HTTPError as err: + if used_urllib is urllib2: + status_code = err.code + if 400 <= status_code <= 499: + _log.warning("URL %s was not found (HTTP response code %s), not trying again" % (url, status_code)) break else: _log.warning("HTTPError occurred while trying to download %s to %s: %s" % (url, path, err)) attempt_cnt += 1 except IOError as err: _log.warning("IOError occurred while trying to download %s to %s: %s" % (url, path, err)) + error_re = re.compile(r"") + if error_re.match(str(err)): + if not HAVE_REQUESTS: + raise EasyBuildError("SSL issues with urllib2. If you are using RHEL/CentOS 6.x please " + "install the python-requests and pyOpenSSL RPM packages and try again.") + _log.info("Downloading using requests package instead of urllib2") + used_urllib = requests attempt_cnt += 1 except Exception, err: raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 33828b63a3..674d23095e 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -438,7 +438,7 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): # sanity check: make sure all patched files are downloaded ec_files = [] - for patched_file in patched_files: + for patched_file in [f for f in patched_files if not f.startswith('test/')]: fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:]) if os.path.exists(os.path.join(path, fn)): ec_files.append(os.path.join(path, fn)) @@ -809,6 +809,19 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ return file_info, deleted_paths, git_repo, pr_branch, diff_stat +def is_patch_for(patch_name, ec): + """Check whether specified patch matches any patch in the provided EasyConfig instance.""" + res = False + for patch in ec['patches']: + if isinstance(patch, (tuple, list)): + patch = patch[0] + if patch == patch_name: + res = True + break + + return res + + def det_patch_specs(patch_paths, file_info): """ Determine software names for patch files """ print_msg("determining software names for patch files...") @@ -819,8 +832,8 @@ def det_patch_specs(patch_paths, file_info): # consider patch lists of easyconfigs being provided for ec in file_info['ecs']: - if patch_file in ec['patches']: - soft_name = ec.name + if is_patch_for(patch_file, ec): + soft_name = ec['name'] break if soft_name: @@ -847,17 +860,6 @@ def find_software_name_for_patch(patch_name): :return: name of the software that this patch file belongs to (if found) """ - def is_patch_for(patch_name, ec): - """Check whether specified patch matches any patch in the provided EasyConfig instance.""" - res = False - for patch in ec['patches']: - if isinstance(patch, (tuple, list)): - patch = patch[0] - if patch == patch_name: - res = True - break - - return res robot_paths = build_option('robot_path') soft_name = None diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 5da7ef2fa2..f1f8e54f4f 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -56,6 +56,7 @@ TESTCASES_STEP = 'testcases' START = 'start' +PARSE = 'parse' END = 'end' PRE_PREF = 'pre_' @@ -67,39 +68,51 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -KNOWN_HOOKS = [h + HOOK_SUFF for h in [START] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]] +HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] + + +# cached version of hooks, to avoid having to load them from file multiple times +_cached_hooks = {} def load_hooks(hooks_path): """Load defined hooks (if any).""" - hooks = {} - - if hooks_path: - if not os.path.exists(hooks_path): - raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path) - - (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1]) - if hooks_file_ext == '.py': - _log.info("Importing hooks implementation from %s...", hooks_path) - try: - # import module that defines hooks, and collect all functions of which name ends with '_hook' - imported_hooks = imp.load_source(hooks_filename, hooks_path) - for attr in dir(imported_hooks): - if attr.endswith(HOOK_SUFF): - hook = getattr(imported_hooks, attr) - if callable(hook): - hooks.update({attr: hook}) - else: - _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) - _log.info("Found hooks: %s", sorted(hooks.keys())) - except ImportError as err: - raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) - else: - raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)") + + if hooks_path in _cached_hooks: + hooks = _cached_hooks[hooks_path] + else: - _log.info("No location for hooks implementation provided, no hooks defined") + hooks = {} + if hooks_path: + if not os.path.exists(hooks_path): + raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path) + + (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1]) + if hooks_file_ext == '.py': + _log.info("Importing hooks implementation from %s...", hooks_path) + try: + # import module that defines hooks, and collect all functions of which name ends with '_hook' + imported_hooks = imp.load_source(hooks_filename, hooks_path) + for attr in dir(imported_hooks): + if attr.endswith(HOOK_SUFF): + hook = getattr(imported_hooks, attr) + if callable(hook): + hooks.update({attr: hook}) + else: + _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) + _log.info("Found hooks: %s", sorted(hooks.keys())) + except ImportError as err: + raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) + else: + raise EasyBuildError("Provided path for hooks implementation should be path to a Python file (*.py)") + else: + _log.info("No location for hooks implementation provided, no hooks defined") - verify_hooks(hooks) + verify_hooks(hooks) + + # cache loaded hooks, so we don't need to load them from file again + _cached_hooks[hooks_path] = hooks return hooks @@ -157,7 +170,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): return res -def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None): +def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): """ Run hook with specified label. @@ -166,6 +179,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None) :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook :param args: arguments to pass to hook function + :param msg: custom message that is printed when hook is called """ hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) if hook: @@ -177,6 +191,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None) elif post_step_hook: label = 'post-' + label - print_msg("Running %s hook..." % label) + if msg is None: + msg = "Running %s hook..." % label + print_msg(msg) + _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) hook(*args) diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 11f1ff40e7..31b7d8181a 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -34,6 +34,7 @@ import re import time +from pkg_resources import get_distribution, DistributionNotFound from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg @@ -51,7 +52,7 @@ from gc3libs import Application, Run, create_engine from gc3libs.core import Engine from gc3libs.quantity import hours as hr - from gc3libs.workflow import DependentTaskCollection + from gc3libs.workflow import AbortOnError, DependentTaskCollection # inject EasyBuild logger into GC3Pie gc3libs.log = fancylogger.getLogger('gc3pie', fname=False) @@ -61,6 +62,16 @@ # instruct GC3Pie to not ignore errors, but raise exceptions instead gc3libs.UNIGNORE_ALL_ERRORS = True + # note: order of class inheritance is important! + class _BuildTaskCollection(AbortOnError, DependentTaskCollection): + """ + A `DependentTaskCollection`:class: that aborts execution upon error. + + This is used to stop the build process in case some dependency + fails. See also ``_ + """ + pass + except ImportError as err: _log.debug("Failed to import gc3libs from GC3Pie." " Silently ignoring, this is a real issue only when GC3Pie is used as backend for --job") @@ -78,8 +89,7 @@ class GC3Pie(JobBackend): terminated. """ - REQ_VERSION = '2.4.0' - VERSION_REGEX = re.compile(r'^(?P\S*) version') + REQ_VERSION = '2.5.0' @only_if_module_is_available('gc3libs', pkgname='gc3pie') def __init__(self, *args, **kwargs): @@ -90,24 +100,15 @@ def __init__(self, *args, **kwargs): @only_if_module_is_available('gc3libs', pkgname='gc3pie') def _check_version(self): """Check whether GC3Pie version complies with required version.""" - # location of __version__ to use may change, depending on the minimal required SVN revision for development versions - version_str = gc3libs.core.__version__ - - match = self.VERSION_REGEX.search(version_str) - if match: - version = match.group('version') - self.log.debug("Parsed GC3Pie version info: '%s'", version) - - if version == 'development': - # presume it's OK -- there's no way to check since GC3Pie switched to git - return True + try: + pkg = get_distribution('gc3pie') + except DistributionNotFound as err: + raise EasyBuildError( + "Cannot load GC3Pie package: %s" % err) - if LooseVersion(version) < LooseVersion(self.REQ_VERSION): - raise EasyBuildError("Found GC3Pie version %s, but version %s or more recent is required", - version, self.REQ_VERSION) - else: - raise EasyBuildError("Failed to parse GC3Pie version string '%s' using pattern %s", - version_str, self.VERSION_REGEX.pattern) + if LooseVersion(pkg.version) < LooseVersion(self.REQ_VERSION): + raise EasyBuildError("Found GC3Pie version %s, but version %s or more recent is required", + pkg.version, self.REQ_VERSION) def init(self): """ @@ -124,7 +125,7 @@ def init(self): self.config_files.append(cfgfile) self.output_dir = build_option('job_output_dir') - self.jobs = DependentTaskCollection(output_dir=self.output_dir) + self.jobs = _BuildTaskCollection(output_dir=self.output_dir) self.job_cnt = 0 # after polling for job status, sleep for this time duration @@ -135,23 +136,20 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None): """ Create and return a job object with the given parameters. - First argument `server` is an instance of the corresponding - `JobBackend` class, i.e., a `GC3Pie`:class: instance in this case. - - Second argument `script` is the content of the job script + Argument *script* is the content of the job script itself, i.e., the sequence of shell commands that will be executed. - Third argument `name` sets the job human-readable name. + Argument *name* sets the job's human-readable name. - Fourth (optional) argument `env_vars` is a dictionary with + Optional argument *env_vars* is a dictionary with key-value pairs of environment variables that should be passed on to the job. - Fifth and sixth (optional) arguments `hours` and `cores` should be + Optional arguments *hours* and *cores* should be integer values: - * hours must be in the range 1 .. MAX_WALLTIME; - * cores depends on which cluster the job is being run. + - *hours* must be in the range 1 .. ``MAX_WALLTIME``; + - *cores* depends on which cluster the job is being run. """ named_args = { 'jobname': name, # job name in GC3Pie @@ -222,21 +220,12 @@ def complete(self): # some sites may not be happy with flooding the cluster with build jobs... self._engine.max_in_flight = build_option('job_max_jobs') - # `Engine.stats()` (which is used later on in `_print_status_report()`) - # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible - # with both - try: - self._engine.init_stats_for(Application) - except AttributeError: - _log.debug("No `init_stats_for` method in the Engine class;" - " assuming pre-2.5.0 GC3Pie and ignoring error.") - # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. self._engine.add(self.jobs) - # in case you want to select a specific resource, call + # select a specific execution resource? target_resource = build_option('job_target_resource') if target_resource: res = self._engine.select_resource(target_resource) @@ -265,10 +254,10 @@ def _print_status_report(self): Print a job status report to STDOUT and the log file. The number of jobs in each state is reported; the - figures are extracted from the `stats()` method of the + figures are extracted from the `counts()` method of the currently-running GC3Pie engine. """ - stats = self._engine.stats(only=Application) + stats = self._engine.counts(only=Application) states = ', '.join(["%d %s" % (stats[s], s.lower()) for s in stats if s != 'total' and stats[s]]) print_msg("GC3Pie job overview: %s (total: %s)" % (states, self.job_cnt), log=self.log, silent=build_option('silent')) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ed8c43702c..fcf5baa035 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -33,6 +33,7 @@ :author: Fotis Georgatos (Uni.Lu, NTUA) :author: Damian Alvarez (Forschungszentrum Juelich GmbH) """ +import copy import os import re import sys @@ -45,7 +46,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file -from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, modules_tool +from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, modules_tool from easybuild.tools.utilities import quote_str @@ -197,6 +198,53 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + def modulerc(self, module_version=None): + """ + Generate contents of .modulerc file, in Tcl syntax (compatible with all module tools, incl. Lmod) + + :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys) + """ + modulerc = [ModuleGeneratorTcl.MODULE_SHEBANG] + + if module_version: + if isinstance(module_version, dict): + expected_keys = ['modname', 'sym_version', 'version'] + if sorted(module_version.keys()) == expected_keys: + + module_version_statement = "module-version %(modname)s %(sym_version)s" + + # for Environment Modules we need to guard the module-version statement, + # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over, + # which occur because the .modulerc is parsed twice + # "module-info version " returns its argument if that argument is not a symbolic version (yet), + # and returns the corresponding real version in case the argument is an existing symbolic version + # cfr. https://sourceforge.net/p/modules/mailman/message/33399425/ + if modules_tool().__class__ == EnvironmentModulesC: + + modname, sym_version, version = [module_version[key] for key in expected_keys] + + # determine module name with symbolic version + if version in modname: + # take a copy so we don't modify original value + module_version = copy.copy(module_version) + module_version['sym_modname'] = modname.replace(version, sym_version) + else: + raise EasyBuildError("Version '%s' does not appear in module name '%s'", version, modname) + + module_version_statement = '\n'.join([ + 'if {"%(sym_modname)s" eq [module-info version %(sym_modname)s]} {', + ' ' * 4 + module_version_statement, + "}", + ]) + + modulerc.append(module_version_statement % module_version) + else: + raise EasyBuildError("Incorrect module_version spec, expected keys: %s", expected_keys) + else: + raise EasyBuildError("Incorrect module_version value type: %s", type(module_version)) + + return '\n'.join(modulerc) + # From this point on just not implemented methods def check_group(self, group, error_msg=None): diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d9a4d9dc9..7550b79dde 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -111,6 +111,7 @@ [^\s\(]*[^:/] # module name must not have '(' or whitespace in it, must not end with ':' or '/' ) # end named group for module name (?P\(default\))? # optional '(default)' that's not part of module name + (\([^()]+\))? # ignore '(...)' that is not part of module name (e.g. for symbolic versions) \s*$ # ignore whitespace at the end of the line """, re.VERBOSE), } @@ -241,16 +242,16 @@ def set_and_check_version(self): if self.REQ_VERSION is not None: self.log.debug("Required minimum version defined.") if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): - raise EasyBuildError("EasyBuild requires v%s >= v%s, found v%s", - self.__class__.__name__, self.version, self.REQ_VERSION) + raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", + self.__class__.__name__, self.REQ_VERSION, self.version) else: self.log.debug('Version %s matches requirement >= %s', self.version, self.REQ_VERSION) if self.MAX_VERSION is not None: self.log.debug("Maximum allowed version defined.") if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): - raise EasyBuildError("EasyBuild requires v%s <= v%s, found v%s", - self.__class__.__name__, self.version, self.MAX_VERSION) + raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", + self.__class__.__name__, self.MAX_VERSION, self.version) else: self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION) @@ -1106,7 +1107,7 @@ class Lmod(ModulesTool): """Interface to Lmod.""" COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' - REQ_VERSION = '5.8' + REQ_VERSION = '6.6.3' REQ_VERSION_DEPENDS_ON = '7.6.1' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 339bf1f33d..85cdab8a32 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -218,6 +218,7 @@ def __init__(self, *args, **kwargs): """Constructor.""" self.with_include = kwargs.pop('with_include', True) + self.single_cfg_level = kwargs.pop('single_cfg_level', False) self.default_repositorypath = [mk_full_default_path('repositorypath')] self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] @@ -568,6 +569,8 @@ def github_options(self): opts = OrderedDict({ 'check-github': ("Check status of GitHub integration, and report back", None, 'store_true', False), + 'check-contrib': ("Runs checks to see whether the given easyconfigs are ready to be contributed back", + None, 'store_true', False), 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), @@ -769,36 +772,10 @@ def postprocess(self): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() - # fail early if required dependencies for functionality requiring using GitHub API are not available: - if self.options.from_pr or self.options.upload_test_report: - if not HAVE_GITHUB_API: - raise EasyBuildError("Required support for using GitHub API is not available (see warnings).") - - if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: - error_msg = "Generating Lua module files requires Lmod as modules tool; " - mod_syntaxes = ', '.join(sorted(avail_module_generators().keys())) - error_msg += "use --module-syntax to specify a different module syntax to use (%s)" % mod_syntaxes - raise EasyBuildError(error_msg) - - # check whether specified action --detect-loaded-modules is valid - if self.options.detect_loaded_modules not in LOADED_MODULES_ACTIONS: - raise EasyBuildError("Unknown action specified to --detect-loaded-modules: %s (known values: %s)", - self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS)) - - # make sure a GitHub token is available when it's required - if self.options.upload_test_report: - if not HAVE_KEYRING: - raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.") - if self.options.github_user is None: - raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.") - token = fetch_github_token(self.options.github_user) - if token is None: - raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user) - - # make sure autopep8 is available when it needs to be - if self.options.dump_autopep8: - if not HAVE_AUTOPEP8: - raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") + # run configuration checks, unless only a single configuration level is being processed + # (this should only happen during --show-config) + if not self.single_cfg_level: + self._postprocess_checks() # imply --terse for --last-log to avoid extra output that gets in the way if self.options.last_log: @@ -859,6 +836,43 @@ def _postprocess_include(self): if self.options.include_toolchains: include_toolchains(self.tmpdir, self.options.include_toolchains) + def _postprocess_checks(self): + """Check whether (combination of) configuration options make sense.""" + + # fail early if required dependencies for functionality requiring using GitHub API are not available: + if self.options.from_pr or self.options.upload_test_report: + if not HAVE_GITHUB_API: + raise EasyBuildError("Required support for using GitHub API is not available (see warnings)") + + # using Lua module syntax only makes sense when modules tool being used is Lmod + if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: + error_msg = "Generating Lua module files requires Lmod as modules tool; " + mod_syntaxes = ', '.join(sorted(avail_module_generators().keys())) + error_msg += "use --module-syntax to specify a different module syntax to use (%s)" % mod_syntaxes + raise EasyBuildError(error_msg) + + # check whether specified action --detect-loaded-modules is valid + if self.options.detect_loaded_modules not in LOADED_MODULES_ACTIONS: + error_msg = "Unknown action specified to --detect-loaded-modules: %s (known values: %s)" + raise EasyBuildError(error_msg % (self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS))) + + # make sure a GitHub token is available when it's required + if self.options.upload_test_report: + if not HAVE_KEYRING: + raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available") + if self.options.github_user is None: + raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token") + token = fetch_github_token(self.options.github_user) + if token is None: + raise EasyBuildError("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) + + # make sure autopep8 is available when it needs to be + if self.options.dump_autopep8: + if not HAVE_AUTOPEP8: + raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") + + self.log.info("Checks on configuration options passed") + def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: @@ -1042,7 +1056,7 @@ def reparse_cfg(args=None, withcfg=True): if args is None: args = [] cfg = EasyBuildOptions(go_args=args, go_useconfigfiles=withcfg, envvar_prefix=CONFIG_ENV_VAR_PREFIX, - with_include=False) + with_include=False, single_cfg_level=True) return cfg.dict_by_prefix() diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index f797aa0a6e..8e40328b60 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -48,8 +48,10 @@ from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version + _log = fancylogger.getLogger('tools.robot', fname=False) + def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_path, auto_robot=False): """Determine robot path.""" robot_path = robot_paths_option[:] @@ -91,14 +93,35 @@ def mk_key(spec): return (spec['name'], det_full_ec_version(spec)) + # determine whether any 'wrappers' are involved + wrapper_deps = {} + for ec in ordered_ecs: + # easyconfigs using ModuleRC install a 'wrapper' for their dependency + # these need to be filtered out to avoid reporting false conflicts... + if ec['ec']['easyblock'] == 'ModuleRC': + wrapper_deps[mk_key(ec)] = mk_key(ec['ec']['dependencies'][0]) + + def mk_dep_keys(deps): + """Create keys for given list of dependencies.""" + res = [] + for dep in deps: + # filter out dependencies marked as external module + if not dep.get('external_module', False): + key = mk_key(dep) + # replace 'wrapper' dependencies with the dependency they're wrapping + if key in wrapper_deps: + key = wrapper_deps[key] + res.append(key) + return res + # construct a dictionary: (name, installver) tuple to (build) dependencies deps_for, dep_of = {}, {} for node in ordered_ecs: node_key = mk_key(node) # exclude external modules, since we can't check conflicts on them (we don't even know the software name) - build_deps = [mk_key(d) for d in node['builddependencies'] if not d.get('external_module', False)] - deps = [mk_key(d) for d in node['ec'].all_dependencies if not d.get('external_module', False)] + build_deps = mk_dep_keys(node['builddependencies']) + deps = mk_dep_keys(node['ec'].all_dependencies) # separate runtime deps from build deps runtime_deps = [d for d in deps if d not in build_deps] @@ -111,8 +134,10 @@ def mk_key(spec): if check_inter_ec_conflicts: # add ghost entry that depends on each of the specified easyconfigs, - # since we want to check for conflicts between specified easyconfigs too - deps_for[(None, None)] = ([], [mk_key(e) for e in easyconfigs]) + # since we want to check for conflicts between specified easyconfigs too; + # 'wrapper' easyconfigs are not included to avoid false conflicts being reported + ec_keys = [k for k in [mk_key(e) for e in easyconfigs] if k not in wrapper_deps] + deps_for[(None, None)] = ([], ec_keys) # iteratively expand list of dependencies last_deps_for = None diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 0a91ad95f7..329a9d3ace 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.6.2') +VERSION = LooseVersion('3.7.0.dev0') UNKNOWN = 'UNKNOWN' diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index bcf69e1bab..30fbb9e8e2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -45,6 +45,8 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax from easybuild.tools.filetools import copy_dir, copy_file, mkdir, read_file, remove_file, write_file +from easybuild.tools.module_generator import module_generator +from easybuild.tools.modules import reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild @@ -1375,12 +1377,95 @@ def test_checksum_step(self): self.assertEqual(stdout, '') self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz") + def test_check_checksums(self): + """Test for check_checksums_for and check_checksums methods.""" + testdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb') + + ec = process_easyconfig(toy_ec)[0] + eb = get_easyblock_instance(ec) + + def run_checks(): + expected = "Checksums missing for one or more sources/patches in toy-0.0-gompi-1.3.12-test.eb: " + expected += "found 1 sources + 1 patches vs 1 checksums" + self.assertEqual(res[0], expected) + self.assertTrue(res[1].startswith("Non-SHA256 checksum found for toy-0.0.tar.gz:")) + + # check for main sources/patches should reveal two issues with checksums + res = eb.check_checksums_for(eb.cfg) + self.assertEqual(len(res), 2) + run_checks() + + # full check also catches checksum issues with extensions + res = eb.check_checksums() + self.assertEqual(len(res), 5) + run_checks() + + idx = 2 + for ext in ['bar', 'barbar', 'toy']: + expected = "Checksums missing for one or more sources/patches of extension %s in " % ext + self.assertTrue(res[idx].startswith(expected)) + idx += 1 + def test_this_is_easybuild(self): """Test 'this_is_easybuild' function (and get_git_revision function used by it).""" # make sure both return a non-Unicode string self.assertTrue(isinstance(get_git_revision(), str)) self.assertTrue(isinstance(this_is_easybuild(), str)) + def test_stale_module_caches(self): + """Test whether module caches are reset between builds.""" + + ec1 = os.path.join(self.test_prefix, 'one.eb') + ec1_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'one'", + "version = '1.0.2'", + "homepage = 'https://example.com'", + "description = '1st test easyconfig'", + "toolchain = {'name': 'dummy', 'version': ''}", + ]) + write_file(ec1, ec1_txt) + + # key aspect here is that two/2.0 depends on one/1.0 (which is an alias for one/1.0.2) + ec2 = os.path.join(self.test_prefix, 'two.eb') + ec2_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'two'", + "version = '2.0'", + "toolchain = {'name': 'dummy', 'version': ''}", + "homepage = 'https://example.com'", + "description = '2nd test easyconfig'", + "dependencies = [('one', '1.0')]", + ]) + write_file(ec2, ec2_txt) + + # populate modules avail/show cache with result for "show one/1.0" when it doesn't exist yet + # need to make sure we use same $MODULEPATH value as the one that is in place during build + moddir = os.path.join(self.test_installpath, 'modules', 'all') + self.modtool.use(moddir) + self.assertFalse(self.modtool.exist(['one/1.0'])[0]) + + # add .modulerc to install version alias one/1.0 for one/1.0.2 + # this makes cached result for "show one/1.0" incorrect as soon as one/1.0.2 is installed via one.eb + modgen = module_generator(None) + module_version_spec = {'modname': 'one/1.0.2', 'sym_version': '1.0', 'version': '1.0.2'} + modulerc_txt = modgen.modulerc(module_version=module_version_spec) + one_moddir = os.path.join(self.test_installpath, 'modules', 'all', 'one') + write_file(os.path.join(one_moddir, '.modulerc'), modulerc_txt) + + # check again, this just grabs the cached results for 'avail one/1.0' & 'show one/1.0' + self.assertFalse(self.modtool.exist(['one/1.0'])[0]) + + # one/1.0 still doesn't exist yet (because underlying one/1.0.2 doesn't exist yet), even after clearing cache + reset_module_caches() + self.assertFalse(self.modtool.exist(['one/1.0'])[0]) + + # installing both one.eb and two.eb in one go should work + # this verifies whether the "module show" cache is cleared in between builds, + # since one/1.0 is required for ec2, and the underlying one/1.0.2 is installed via ec1 in the same session + self.eb_main([ec1, ec2], raise_error=True, do_build=True, verbose=True) + def suite(): """ return all the tests in this file """ diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 147616795d..48e51028f4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -48,12 +48,13 @@ from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig, create_paths, copy_easyconfigs from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path, letter_dir_for -from easybuild.framework.easyconfig.easyconfig import process_easyconfig, resolve_template, verify_easyconfig_filename +from easybuild.framework.easyconfig.easyconfig import process_easyconfig, resolve_template +from easybuild.framework.easyconfig.easyconfig import det_subtoolchain_version, verify_easyconfig_filename from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph, get_paths_for -from easybuild.framework.easyconfig.tools import find_related_easyconfigs, parse_easyconfigs +from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph +from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes @@ -66,6 +67,7 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str from test.framework.utilities import find_full_path @@ -203,26 +205,46 @@ def test_dependency(self): 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', + 'versionsuffix = "-test"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', - 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', - 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', + 'dependencies = [' + ' ("first", "1.1"),' + ' {"name": "second", "version": "2.2"},', + # funky way of referring to version(suffix), but should work! + ' ("foo", "%(version)s", versionsuffix),', + ' ("bar", "1.2.3", "%(versionsuffix)s-123"),', + ']', + 'builddependencies = [', + ' ("first", "1.1"),', + ' {"name": "second", "version": "2.2"},', + ']', ]) self.prep() eb = EasyConfig(self.eb_file) # should include builddependencies - self.assertEqual(len(eb.dependencies()), 4) + self.assertEqual(len(eb.dependencies()), 6) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") - self.assertEqual(second['name'], "second") - self.assertEqual(first['version'], "1.1") + self.assertEqual(first['versionsuffix'], '') + + self.assertEqual(second['name'], "second") self.assertEqual(second['version'], "2.2") + self.assertEqual(second['versionsuffix'], '') + + self.assertEqual(eb['dependencies'][2]['name'], 'foo') + self.assertEqual(eb['dependencies'][2]['version'], '3.14') + self.assertEqual(eb['dependencies'][2]['versionsuffix'], '-test') + + self.assertEqual(eb['dependencies'][3]['name'], 'bar') + self.assertEqual(eb['dependencies'][3]['version'], '1.2.3') + self.assertEqual(eb['dependencies'][3]['versionsuffix'], '-test-123') self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') @@ -1910,7 +1932,7 @@ def test_template_constant_dict(self): expected = { 'bitbucket_account': 'gzip', - 'github_account': None, + 'github_account': 'gzip', 'name': 'gzip', 'nameletter': 'g', 'toolchain_name': 'goolf', @@ -1930,7 +1952,7 @@ def test_template_constant_dict(self): expected = { 'bitbucket_account': 'toy', - 'github_account': None, + 'github_account': 'toy', 'name': 'toy', 'nameletter': 't', 'toolchain_name': 'dummy', @@ -2078,6 +2100,50 @@ def test_resolve_template(self): # '%(name)' is not a correct template spec (missing trailing 's') self.assertEqual(resolve_template('%(name)', tmpl_dict), '%(name)') + def test_det_subtoolchain_version(self): + """Test det_subtoolchain_version function""" + _, all_tc_classes = search_toolchain('') + subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) + optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) + + current_tc = {'name': 'goolfc', 'version': '2.6.10'} + # missing gompic and double golfc should both give exceptions + cands = [{'name': 'golfc', 'version': '2.6.10'}, + {'name': 'golfc', 'version': '2.6.11'}] + self.assertErrorRegex(EasyBuildError, + "No version found for subtoolchain gompic in dependencies of goolfc", + det_subtoolchain_version, current_tc, 'gompic', optional_toolchains, cands) + self.assertErrorRegex(EasyBuildError, + "Multiple versions of golfc found in dependencies of toolchain goolfc: 2.6.10, 2.6.11", + det_subtoolchain_version, current_tc, 'golfc', optional_toolchains, cands) + + # missing candidate for golfc, ok for optional + cands = [{'name': 'gompic', 'version': '2.6.10'}] + versions = [det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands) + for subtoolchain_name in subtoolchains[current_tc['name']]] + self.assertEqual(versions, ['2.6.10', None]) + + # 'dummy', 'dummy' should be ok: return None for GCCcore, and None or '' for 'dummy'. + current_tc = {'name': 'GCC', 'version': '4.8.2'} + cands = [{'name': 'dummy', 'version': 'dummy'}] + versions = [det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands) + for subtoolchain_name in subtoolchains[current_tc['name']]] + self.assertEqual(versions, [None, None]) + + init_config(build_options={ + 'add_dummy_to_minimal_toolchains': True}) + + versions = [det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands) + for subtoolchain_name in subtoolchains[current_tc['name']]] + self.assertEqual(versions, [None, '']) + + # and GCCcore if existing too + current_tc = {'name': 'GCC', 'version': '4.9.3-2.25'} + cands = [{'name': 'GCCcore', 'version': '4.9.3'}] + versions = [det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands) + for subtoolchain_name in subtoolchains[current_tc['name']]] + self.assertEqual(versions, ['4.9.3', '']) + def test_verify_easyconfig_filename(self): """Test verify_easyconfig_filename function""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -2183,6 +2249,68 @@ def test_not_an_easyconfig(self): error_pattern = "Parsing easyconfig file failed: invalid syntax" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, not_an_ec) + def test_check_sha256_checksums(self): + """Test for check_sha256_checksums function.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + checksums_regex = re.compile('^checksums = \[\[(.|\n)*\]\]', re.M) + + # wipe out specified checksums, to make check fail + test_ec = os.path.join(self.test_prefix, 'toy-0.0-fail.eb') + write_file(test_ec, checksums_regex.sub('checksums = []', toy_ec_txt)) + ecs, _ = parse_easyconfigs([(test_ec, False)]) + ecs = [ec['ec'] for ec in ecs] + + # result is non-empty list with strings describing checksum issues + res = check_sha256_checksums(ecs) + # result should be non-empty, i.e. contain a list of messages highlighting checksum issues + self.assertTrue(res) + self.assertTrue(res[0].startswith('Checksums missing for one or more sources/patches in toy-0.0-fail.eb')) + + # test use of whitelist regex patterns: check passes because easyconfig is whitelisted by filename + for regex in ['toy-.*', '.*-0\.0-fail\.eb']: + res = check_sha256_checksums(ecs, whitelist=[regex]) + self.assertFalse(res) + + # re-test with MD5 checksum to make test fail + toy_md5 = 'be662daa971a640e40be5c804d9d7d10' + test_ec_txt = checksums_regex.sub('checksums = ["%s"]' % toy_md5, toy_ec_txt) + + test_ec = os.path.join(self.test_prefix, 'toy-0.0-md5.eb') + write_file(test_ec, test_ec_txt) + ecs, _ = parse_easyconfigs([(test_ec, False)]) + ecs = [ec['ec'] for ec in ecs] + + res = check_sha256_checksums(ecs) + self.assertTrue(res) + self.assertTrue(res[-1].startswith("Non-SHA256 checksum found for toy-0.0.tar.gz")) + + # re-test with right checksum in place + toy_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' + test_ec_txt = checksums_regex.sub('checksums = ["%s"]' % toy_sha256, toy_ec_txt) + test_ec_txt = re.sub('patches = \[(.|\n)*\]', '', test_ec_txt) + + test_ec = os.path.join(self.test_prefix, 'toy-0.0-ok.eb') + write_file(test_ec, test_ec_txt) + ecs, _ = parse_easyconfigs([(test_ec, False)]) + ecs = [ec['ec'] for ec in ecs] + + # if no checksum issues are found, result is an empty list + self.assertEqual(check_sha256_checksums(ecs), []) + + # also test toy easyconfig with extensions, for which some checksums are missing + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb') + ecs, _ = parse_easyconfigs([(toy_ec, False)]) + ecs = [ec['ec'] for ec in ecs] + + # checksum issues found, so result is non-empty + res = check_sha256_checksums(ecs) + self.assertTrue(res) + # multiple checksums listed for source tarball, while exactly one (SHA256) checksum is expected + self.assertTrue(res[1].startswith("Non-SHA256 checksum found for toy-0.0.tar.gz: ")) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index b8ad4cbfa2..0a1bcb19f9 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -137,7 +137,7 @@ def test_v20_deps(self): # name, version, versionsuffix, toolchain ('GCC', '4.7.2', None, None), ('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}), - ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}), + ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'GCC', 'version': '4.7.2'}), ('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}), ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}), ] diff --git a/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.7.2-serial.eb b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.7.2-serial.eb new file mode 100644 index 0000000000..7b2a872702 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.7.2-serial.eb @@ -0,0 +1,35 @@ +easyblock = 'ConfigureMake' + +name = 'FFTW' +version = '3.3.3' +versionsuffix = '-serial' + +homepage = 'http://www.fftw.org' +description = """FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) + in one or more dimensions, of arbitrary input size, and of both real and complex data.""" + +toolchain = {'name': 'GCC', 'version': '4.7.2'} +toolchainopts = {'optarch': True, 'pic': True} + +sources = [SOURCELOWER_TAR_GZ] +source_urls = [homepage] + +common_configopts = "--enable-openmp --with-pic" + +configopts = [ + common_configopts + " --enable-single --enable-sse2", + common_configopts + " --enable-long-double", + common_configopts + " --enable-quad-precision", + common_configopts + " --enable-sse2", # default as last +] + +sanity_check_paths = { + 'files': ['bin/fftw%s' % x for x in ['-wisdom', '-wisdom-to-conf', 'f-wisdom', 'l-wisdom', 'q-wisdom']] + + ['include/fftw3%s' % x for x in ['.f', '.f03', + '.h', 'l.f03', 'q.f03']] + + ['lib/libfftw3%s.a' % x for x in ['', '_omp', 'f', 'f_omp', + 'l', 'l_omp', 'q', 'q_omp']], + 'dirs': ['lib/pkgconfig'], +} + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.8.2-serial.eb b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.8.2-serial.eb new file mode 100644 index 0000000000..a3edcc7a42 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-GCC-4.8.2-serial.eb @@ -0,0 +1,35 @@ +easyblock = 'ConfigureMake' + +name = 'FFTW' +version = '3.3.3' +versionsuffix = '-serial' + +homepage = 'http://www.fftw.org' +description = """FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) + in one or more dimensions, of arbitrary input size, and of both real and complex data.""" + +toolchain = {'name': 'GCC', 'version': '4.7.2'} +toolchainopts = {'optarch': True, 'pic': True} + +sources = [SOURCELOWER_TAR_GZ] +source_urls = [homepage] + +common_configopts = "--enable-threads --enable-openmp --with-pic" + +configopts = [ + common_configopts + " --enable-single --enable-sse2 --enable-mpi", + common_configopts + " --enable-long-double --enable-mpi", + common_configopts + " --enable-quad-precision", + common_configopts + " --enable-sse2 --enable-mpi", # default as last +] + +sanity_check_paths = { + 'files': ['bin/fftw%s' % x for x in ['-wisdom', '-wisdom-to-conf', 'f-wisdom', 'l-wisdom', 'q-wisdom']] + + ['include/fftw3%s' % x for x in ['-mpi.f03', '-mpi.h', '.f', '.f03', + '.h', 'l-mpi.f03', 'l.f03', 'q.f03']] + + ['lib/libfftw3%s%s.a' % (x, y) for x in ['', 'f', 'l'] for y in ['', '_mpi', '_omp', '_threads']] + + ['lib/libfftw3q.a', 'lib/libfftw3q_omp.a'], + 'dirs': ['lib/pkgconfig'], +} + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-gompic-2.6.10.eb b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-gompic-2.6.10.eb new file mode 100644 index 0000000000..52dc0f0662 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/f/FFTW/FFTW-3.3.3-gompic-2.6.10.eb @@ -0,0 +1,34 @@ +easyblock = 'ConfigureMake' + +name = 'FFTW' +version = '3.3.3' + +homepage = 'http://www.fftw.org' +description = """FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) + in one or more dimensions, of arbitrary input size, and of both real and complex data.""" + +toolchain = {'name': 'gompic', 'version': '2.6.10'} +toolchainopts = {'optarch': True, 'pic': True} + +sources = [SOURCELOWER_TAR_GZ] +source_urls = [homepage] + +common_configopts = "--enable-threads --enable-openmp --with-pic" + +configopts = [ + common_configopts + " --enable-single --enable-sse2 --enable-mpi", + common_configopts + " --enable-long-double --enable-mpi", + common_configopts + " --enable-quad-precision", + common_configopts + " --enable-sse2 --enable-mpi", # default as last +] + +sanity_check_paths = { + 'files': ['bin/fftw%s' % x for x in ['-wisdom', '-wisdom-to-conf', 'f-wisdom', 'l-wisdom', 'q-wisdom']] + + ['include/fftw3%s' % x for x in ['-mpi.f03', '-mpi.h', '.f', '.f03', + '.h', 'l-mpi.f03', 'l.f03', 'q.f03']] + + ['lib/libfftw3%s%s.a' % (x, y) for x in ['', 'f', 'l'] for y in ['', '_mpi', '_omp', '_threads']] + + ['lib/libfftw3q.a', 'lib/libfftw3q_omp.a'], + 'dirs': ['lib/pkgconfig'], +} + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/g/gcccuda/gcccuda-2.6.10.eb b/test/framework/easyconfigs/test_ecs/g/gcccuda/gcccuda-2.6.10.eb new file mode 100644 index 0000000000..f709c63df4 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/gcccuda/gcccuda-2.6.10.eb @@ -0,0 +1,21 @@ +easyblock = "Toolchain" + +name = 'gcccuda' +version = '2.6.10' + +homepage = '(none)' +description = """GNU Compiler Collection (GCC) based compiler toolchain, along with CUDA toolkit.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_ver = '4.8.2' +comp = (comp_name, comp_ver) + +# compiler toolchain dependencies +dependencies = [ + comp, + ('CUDA', '5.5.22', '', comp), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/g/golf/golf-1.4.10.eb b/test/framework/easyconfigs/test_ecs/g/golf/golf-1.4.10.eb new file mode 100644 index 0000000000..475b072d51 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/golf/golf-1.4.10.eb @@ -0,0 +1,27 @@ +easyblock = "Toolchain" + +name = 'golf' +version = '1.4.10' + +homepage = '(none)' +description = """GNU Compiler Collection (GCC) based compiler toolchain, including +OpenBLAS (BLAS and LAPACK support), and FFTW.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_version = '4.7.2' +comp = (comp_name, comp_version) + +blaslib = 'OpenBLAS' +blasver = '0.2.6' +blas_suff = '-LAPACK-3.4.2' + +# compiler toolchain dependencies +dependencies = [ + comp, + (blaslib, blasver, blas_suff, comp), + ('FFTW', '3.3.3', '-serial', comp), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/g/golf/golf-2.6.10.eb b/test/framework/easyconfigs/test_ecs/g/golf/golf-2.6.10.eb new file mode 100644 index 0000000000..f571347e8d --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/golf/golf-2.6.10.eb @@ -0,0 +1,27 @@ +easyblock = "Toolchain" + +name = 'golf' +version = '2.6.10' + +homepage = '(none)' +description = """GNU Compiler Collection (GCC) based compiler toolchain, including +OpenBLAS (BLAS and LAPACK support), and FFTW.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_ver = '4.8.2' +comp = (comp_name, comp_ver) + +blaslib = 'OpenBLAS' +blasver = '0.2.8' +blassuff = '-LAPACK-3.4.2' + +# compiler toolchain dependencies +dependencies = [ + comp, + (blaslib, blasver, blassuff, comp), + ('FFTW', '3.3.3', '-serial', comp), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/g/golfc/golfc-2.6.10.eb b/test/framework/easyconfigs/test_ecs/g/golfc/golfc-2.6.10.eb new file mode 100644 index 0000000000..0dff5bc15b --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/golfc/golfc-2.6.10.eb @@ -0,0 +1,30 @@ +easyblock = "Toolchain" + +name = 'golfc' +version = '2.6.10' + +homepage = '(none)' +description = """GCC based compiler toolchain __with CUDA support__, and including + OpenBLAS (BLAS and LAPACK support) and FFTW.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_ver = '4.8.2' +comp = (comp_name, comp_ver) + +blaslib = 'OpenBLAS' +blasver = '0.2.8' +blassuff = '-LAPACK-3.4.2' + +# compiler toolchain dependencies +# we need GCC as explicit dependency instead of golf toolchain +# because of toolchain preperation functions +dependencies = [ + comp, # part of golf and gcccuda + ('CUDA', '5.5.22', '', comp), # part of gcccuda + (blaslib, blasver, blassuff, comp), + ('FFTW', '3.3.3', '-serial', comp), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/g/gompic/gompic-2.6.10.eb b/test/framework/easyconfigs/test_ecs/g/gompic/gompic-2.6.10.eb new file mode 100644 index 0000000000..ee4c42cd44 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/gompic/gompic-2.6.10.eb @@ -0,0 +1,23 @@ +easyblock = "Toolchain" + +name = 'gompic' +version = '2.6.10' + +homepage = '(none)' +description = """GNU Compiler Collection (GCC) based compiler toolchain along with CUDA toolkit, + including OpenMPI for MPI support with CUDA features enabled.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_ver = '4.8.2' +comp = (comp_name, comp_ver) + +# compiler toolchain dependencies +dependencies = [ + comp, # part of gcccuda + ('CUDA', '5.5.22', '', comp), # part of gcccuda + ('OpenMPI', '1.7.3', '', ('gcccuda', version)), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/g/goolf/goolf-1.4.10.eb b/test/framework/easyconfigs/test_ecs/g/goolf/goolf-1.4.10.eb index c0de684f39..ba1f5dc1ea 100644 --- a/test/framework/easyconfigs/test_ecs/g/goolf/goolf-1.4.10.eb +++ b/test/framework/easyconfigs/test_ecs/g/goolf/goolf-1.4.10.eb @@ -29,7 +29,7 @@ comp_mpi_tc = (comp_mpi_tc_name, comp_mpi_tc_ver) dependencies = [ comp, ('OpenMPI', '1.6.4', '', comp), # part of gompi-1.4.10 - (blaslib, blasver, blas_suff, comp_mpi_tc), + (blaslib, blasver, blas_suff, comp), ('FFTW', '3.3.3', '', comp_mpi_tc), ('ScaLAPACK', '2.0.2', '-%s%s' % (blas, blas_suff), comp_mpi_tc) ] diff --git a/test/framework/easyconfigs/test_ecs/g/goolfc/goolfc-2.6.10.eb b/test/framework/easyconfigs/test_ecs/g/goolfc/goolfc-2.6.10.eb new file mode 100644 index 0000000000..2efc143c60 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/goolfc/goolfc-2.6.10.eb @@ -0,0 +1,38 @@ +easyblock = "Toolchain" + +name = 'goolfc' +version = '2.6.10' + +homepage = '(none)' +description = """GCC based compiler toolchain __with CUDA support__, and including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +comp_name = 'GCC' +comp_ver = '4.8.2' +comp = (comp_name, comp_ver) + +# toolchain used to build goolfc dependencies +comp_mpi_tc_name = 'gompic' +comp_mpi_tc_ver = version +comp_mpi_tc = (comp_mpi_tc_name, comp_mpi_tc_ver) + +blaslib = 'OpenBLAS' +blasver = '0.2.8' +blassuff = '-LAPACK-3.4.2' +blas = '-%s-%s%s' % (blaslib, blasver, blassuff) + +# compiler toolchain dependencies +# we need GCC and OpenMPI as explicit dependencies instead of gompi toolchain +# because of toolchain preperation functions +dependencies = [ + comp, # part of gompic + ('CUDA', '5.5.22', '', comp), # part of gompic + ('OpenMPI', '1.7.3', '', ('gcccuda', version)), # part of gompic + (blaslib, blasver, blassuff, comp), + ('FFTW', '3.3.3', '', comp_mpi_tc), + ('ScaLAPACK', '2.0.2', blas, comp_mpi_tc), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.8-gcccuda-2.6.10.eb b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.8-gcccuda-2.6.10.eb new file mode 100644 index 0000000000..b61501e141 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.8-gcccuda-2.6.10.eb @@ -0,0 +1,19 @@ +easyblock = 'ConfigureMake' + +name = 'hwloc' +version = "1.8" + +homepage = 'http://www.open-mpi.org/projects/hwloc/' +description = """The Portable Hardware Locality (hwloc) software package provides a portable abstraction + (across OS, versions, architectures, ...) of the hierarchical topology of modern architectures, including + NUMA memory nodes, sockets, shared caches, cores and simultaneous multithreading. It also gathers various + system attributes such as cache and memory information as well as the locality of I/O devices such as + network interfaces, InfiniBand HCAs or GPUs. It primarily aims at helping applications with gathering + information about modern computing hardware so as to exploit it accordingly and efficiently.""" + +toolchain = {'name': 'gcccuda', 'version': '2.6.10'} + +source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/'] +sources = [SOURCE_TAR_GZ] + +moduleclass = 'system' diff --git a/test/framework/easyconfigs/test_ecs/i/iccifort/iccifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/test_ecs/i/iccifort/iccifort-2013.5.192-GCC-4.8.3.eb index 9a152f81fe..05baba470c 100644 --- a/test/framework/easyconfigs/test_ecs/i/iccifort/iccifort-2013.5.192-GCC-4.8.3.eb +++ b/test/framework/easyconfigs/test_ecs/i/iccifort/iccifort-2013.5.192-GCC-4.8.3.eb @@ -10,8 +10,9 @@ description = """Intel C, C++ and Fortran compilers""" toolchain = {'name': 'dummy', 'version': 'dummy'} dependencies = [ - ('icc', version, versionsuffix), - ('ifort', version, versionsuffix), + # use of %(version)s & %(versionsuffix)s is a bit silly here, but it should work too... + ('icc', version, '%(versionsuffix)s'), + ('ifort', '%(version)s', versionsuffix), ] moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb similarity index 96% rename from test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb rename to test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb index 1e915a802f..1ea03f2c30 100644 --- a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb @@ -9,7 +9,7 @@ versionsuffix = '-LAPACK-%s' % lapackver homepage = 'http://xianyi.github.com/OpenBLAS/' description = """OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.""" -toolchain = {'name': 'gompi', 'version': '1.4.10'} +toolchain = {'name': 'GCC', 'version': '4.7.2'} lapack_src = 'lapack-%s.tgz' % lapackver large_src = 'large.tgz' diff --git a/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb new file mode 100644 index 0000000000..18d9a007ef --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb @@ -0,0 +1,53 @@ +easyblock = 'ConfigureMake' + +name = 'OpenBLAS' +version = '0.2.8' + +lapackver = '3.4.2' +versionsuffix = '-LAPACK-%s' % lapackver + +homepage = 'http://xianyi.github.com/OpenBLAS/' +description = """OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.""" + +toolchain = {'name': 'GCC', 'version': '4.8.2'} + +lapack_src = 'lapack-%s.tgz' % lapackver +large_src = 'large.tgz' +timing_src = 'timing.tgz' +sources = [ + 'v%(version)s.tar.gz', + lapack_src, + large_src, + timing_src, +] +source_urls = [ + # order matters, trying to download the LAPACK tarball from GitHub causes trouble + "http://www.netlib.org/lapack/", + "http://www.netlib.org/lapack/timing/", + "https://github.com/xianyi/OpenBLAS/archive/", +] + +patches = [ + # 'OpenBLAS-%s_Makefile-LAPACK-sources.patch' % version, + (lapack_src, '.'), # copy LAPACK tarball to unpacked OpenBLAS dir + (large_src, '.'), + (timing_src, '.'), +] + +skipsteps = ['configure'] + +threading = 'USE_THREAD=1' +buildopts = 'BINARY=64 ' + threading + ' CC="$CC" FC="$F77"' +installopts = threading + " PREFIX=%(installdir)s" + +# extensive testing can be enabled by uncommenting the line below +#runtest = 'PATH=.:$PATH lapack-timing' + +sanity_check_paths = { + 'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h', + 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h', + 'lib/libopenblas.a', 'lib/libopenblas.%s' % SHLIB_EXT], + 'dirs': [], +} + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-1.7.3-gcccuda-2.6.10.eb b/test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-1.7.3-gcccuda-2.6.10.eb new file mode 100644 index 0000000000..fdbc43e64d --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-1.7.3-gcccuda-2.6.10.eb @@ -0,0 +1,37 @@ +easyblock = 'ConfigureMake' + +name = 'OpenMPI' +version = "1.7.3" + +homepage = 'http://www.open-mpi.org/' +description = """The Open MPI Project is an open source MPI-2 implementation.""" + +toolchain = {'name': 'gcccuda', 'version': '2.6.10'} + +sources = [SOURCELOWER_TAR_GZ] +source_urls = ['http://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads'] + +patches = [ + 'OpenMPI-%(version)s_common-cuda-lib.patch', + 'OpenMPI-%(version)s-vt_cupti_events.patch', +] + +dependencies = [('hwloc', '1.8')] + +configopts = '--with-threads=posix --enable-shared --enable-mpi-thread-multiple --with-verbs ' +configopts += '--enable-mpirun-prefix-by-default ' # suppress failure modes in relation to mpirun path +configopts += '--with-hwloc=$EBROOTHWLOC ' # hwloc support +configopts += '--with-cuda=$CUDA_HOME ' # CUDA-aware build; N.B. --disable-dlopen is incompatible + +# needed for --with-verbs +osdependencies = [('libibverbs-dev', 'libibverbs-devel')] + +libs = ["mpi_cxx", "mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte", "vt", "vt-hyb", "vt-mpi", "vt-mpi-unify"] +sanity_check_paths = { + 'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] + + ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in libs] + + ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]], + 'dirs': ["include/openmpi/ompi/mpi/cxx"], +} + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb index 238a910851..a87029e69d 100644 --- a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb @@ -20,7 +20,7 @@ blassuff = '-LAPACK-3.4.2' versionsuffix = "-%s-%s%s" % (blaslib, blasver, blassuff) -dependencies = [(blaslib, blasver, blassuff)] +dependencies = [(blaslib, blasver, blassuff, ('GCC', '4.7.2'))] # parallel build tends to fail, so disabling it parallel = 1 diff --git a/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2.6.10-OpenBLAS-0.2.8-LAPACK-3.4.2.eb b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2.6.10-OpenBLAS-0.2.8-LAPACK-3.4.2.eb new file mode 100644 index 0000000000..2dc9009a55 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompic-2.6.10-OpenBLAS-0.2.8-LAPACK-3.4.2.eb @@ -0,0 +1,28 @@ +# should be EB_ScaLAPACK, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'ScaLAPACK' +version = '2.0.2' + +homepage = 'http://www.netlib.org/scalapack/' +description = """The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers.""" + +toolchain = {'name': 'gompic', 'version': '2.6.10'} +toolchainopts = {'pic': True} + +source_urls = [homepage] +sources = ['%(namelower)s-%(version)s.tgz'] + +blaslib = 'OpenBLAS' +blasver = '0.2.8' +blassuff = '-LAPACK-3.4.2' + +versionsuffix = "-%s-%s%s" % (blaslib, blasver, blassuff) + +dependencies = [(blaslib, blasver, blassuff)] + +# parallel build tends to fail, so disabling it +parallel = 1 + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index bb01473b53..f70502b501 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -25,6 +25,7 @@ exts_default_options = { } exts_list = [ + 'ls', # extension that is part of "standard library" ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'], # checksum for diff --git a/test/framework/easyconfigs/v2.0/goolf.eb b/test/framework/easyconfigs/v2.0/goolf.eb index 346d5ab55c..e38c8b0d13 100644 --- a/test/framework/easyconfigs/v2.0/goolf.eb +++ b/test/framework/easyconfigs/v2.0/goolf.eb @@ -25,6 +25,6 @@ toolchains = dummy == dummy [DEPENDENCIES] GCC = 4.7.2 OpenMPI = 1.6.4; GCC == 4.7.2 -OpenBLAS = 0.2.6 suffix:-LAPACK-3.4.2; gompi == 1.4.10 +OpenBLAS = 0.2.6 suffix:-LAPACK-3.4.2; GCC == 4.7.2 FFTW = 3.3.3; gompi == 1.4.10 ScaLAPACK = 2.0.2 suffix:-OpenBLAS-0.2.6-LAPACK-3.4.2; gompi == 1.4.10 diff --git a/test/framework/easyconfigs/yeb/goolf-1.4.10.yeb b/test/framework/easyconfigs/yeb/goolf-1.4.10.yeb index e0c502d6b8..1217d5fcfa 100644 --- a/test/framework/easyconfigs/yeb/goolf-1.4.10.yeb +++ b/test/framework/easyconfigs/yeb/goolf-1.4.10.yeb @@ -34,7 +34,7 @@ dependencies: toolchain: *comp - *blaslib: *blasver versionsuffix: *blas_suff - toolchain: *comp_mpi_tc + toolchain: *comp - FFTW: 3.3.3 toolchain: *comp_mpi_tc - ScaLAPACK: 2.0.2 diff --git a/test/framework/filetools.py b/test/framework/filetools.py index bb17cf1680..388b67e27e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -59,6 +59,18 @@ class FileToolsTest(EnhancedTestCase): ('0_foo+0x0x#-$__', 'EB_0_underscore_foo_plus_0x0x_hash__minus__dollar__underscore__underscore_'), ] + def setUp(self): + """Test setup.""" + super(FileToolsTest, self).setUp() + + self.orig_filetools_urllib2_urlopen = ft.urllib2.urlopen + + def tearDown(self): + """Cleanup.""" + super(FileToolsTest, self).tearDown() + + ft.urllib2.urlopen = self.orig_filetools_urllib2_urlopen + def test_extract_cmd(self): """Test various extract commands.""" tests = [ @@ -317,7 +329,7 @@ def test_download_file(self): opts = init_config(args=['--download-timeout=5.3']) init_config(build_options={'download_timeout': opts.download_timeout}) target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') - url = 'https://jenkins1.ugent.be/robots.txt' + url = 'https://raw.githubusercontent.com/easybuilders/easybuild-framework/master/README.rst' try: urllib2.urlopen(url) res = ft.download_file(fn, url, target_location) @@ -349,6 +361,30 @@ def test_download_file(self): self.assertTrue(os.path.exists(target_location)) self.assertTrue(os.path.samefile(path, target_location)) + def test_download_file_requests_fallback(self): + """Test fallback to requests in download_file function.""" + url = 'https://raw.githubusercontent.com/easybuilders/easybuild-framework/master/README.rst' + fn = 'README.rst' + target = os.path.join(self.test_prefix, fn) + + # replace urllib2.urlopen with function that raises SSL error + def fake_urllib2_open(*args, **kwargs): + error_msg = "'], msg="Running parse hook for example.eb...") run_hook('configure', hooks, pre_step_hook=True, args=[None]) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) @@ -118,6 +142,8 @@ def test_run_hook(self): expected_stdout = '\n'.join([ "== Running start hook...", "this is triggered at the very beginning", + "== Running parse hook for example.eb...", + "Parse hook with argument ", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 88ff06d98b..287f766eea 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -45,7 +45,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import Lmod +from easybuild.tools.modules import EnvironmentModulesC, Lmod from easybuild.tools.utilities import quote_str from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config @@ -315,6 +315,46 @@ def test_load(self): init_config(build_options={'mod_depends_on': 'True'}) self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name") + def test_modulerc(self): + """Test modulerc method.""" + self.assertErrorRegex(EasyBuildError, "Incorrect module_version value type", self.modgen.modulerc, 'foo') + + arg = {'foo': 'bar'} + error_pattern = "Incorrect module_version spec, expected keys" + self.assertErrorRegex(EasyBuildError, error_pattern, self.modgen.modulerc, arg) + + modulerc = self.modgen.modulerc({'modname': 'test/1.2.3.4.5', 'sym_version': '1.2.3', 'version': '1.2.3.4.5'}) + + if self.modtool.__class__ == EnvironmentModulesC: + expected = '\n'.join([ + '#%Module', + 'if {"test/1.2.3" eq [module-info version test/1.2.3]} {', + ' module-version test/1.2.3.4.5 1.2.3', + '}', + ]) + else: + expected = '\n'.join([ + '#%Module', + "module-version test/1.2.3.4.5 1.2.3", + ]) + + self.assertEqual(modulerc, expected) + + write_file(os.path.join(self.test_prefix, 'test', '1.2.3.4.5'), '#%Module') + write_file(os.path.join(self.test_prefix, 'test', '.modulerc'), modulerc) + + self.modtool.use(self.test_prefix) + + # 'show' picks up on symbolic versions, regardless of modules tool being used + self.assertEqual(self.modtool.exist(['test/1.2.3.4.5', 'test/1.2.3.4', 'test/1.2.3']), [True, False, True]) + + # loading of module with symbolic version works + self.modtool.load(['test/1.2.3']) + # test/1.2.3.4.5 is actually loaded (rather than test/1.2.3) + res = self.modtool.list() + self.assertEqual(len(res), 1) + self.assertEqual(res[0]['mod_name'], 'test/1.2.3.4.5') + def test_unload(self): """Test unload part in generated module file.""" diff --git a/test/framework/options.py b/test/framework/options.py index e51ad805ef..b9b075b584 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -240,11 +240,11 @@ def test_skip(self): # use toy-0.0.eb easyconfig file that comes with the tests topdir = os.path.abspath(os.path.dirname(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') # check log message with --skip for existing module args = [ - eb_file, + toy_ec, '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, @@ -266,7 +266,7 @@ def test_skip(self): # check log message with --skip for non-existing module args = [ - eb_file, + toy_ec, '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, @@ -286,6 +286,31 @@ def test_skip(self): not_found = re.search(not_found_msg, outtxt) self.assertTrue(not_found, "Module not found message there with --skip for non-existing modules: %s" % outtxt) + toy_mod_glob = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '*') + for toy_mod in glob.glob(toy_mod_glob): + remove_file(toy_mod) + self.assertFalse(glob.glob(toy_mod_glob)) + + # make sure that sanity check is *NOT* skipped under --skip + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + regex = re.compile("sanity_check_paths = \{(.|\n)*\}", re.M) + test_ec_txt = regex.sub("sanity_check_paths = {'files': ['bin/nosuchfile'], 'dirs': []}", test_ec_txt) + write_file(test_ec, test_ec_txt) + args = [ + test_ec, + '--skip', + '--force', + ] + error_pattern = "Sanity check failed: no file found at 'bin/nosuchfile'" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + + # check use of skipsteps to skip sanity check + test_ec_txt += "\nskipsteps = ['sanitycheck']\n" + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + + self.assertEqual(len(glob.glob(toy_mod_glob)), 1) def test_job(self): """Test submitting build as a job.""" @@ -558,6 +583,7 @@ def test_list_easyblocks(self): r'\| \|-- DummyExtension', r'\| \|-- EB_toy', r'\| \|-- Toy_Extension', + r'\|-- ModuleRC', r'\|-- Toolchain', r'Extension', r'\|-- ExtensionEasyBlock', @@ -825,8 +851,8 @@ def test_try_robot_force(self): # GCC/OpenMPI dependencies are there, but part of toolchain => 'x' ("GCC-4.6.4.eb", "GCC/4.6.4", 'x'), ("OpenMPI-1.6.4-GCC-4.6.4.eb", "OpenMPI/1.6.4-GCC-4.6.4", 'x'), - # OpenBLAS dependency is there, but not listed => 'x' - ("OpenBLAS-0.2.6-gompi-1.3.12-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-gompi-1.3.12-LAPACK-3.4.2", 'x'), + # OpenBLAS dependency is listed, but not there => ' ' + ("OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2", ' '), # both FFTW and ScaLAPACK are listed => 'F' ("ScaLAPACK-%s.eb" % scalapack_ver, "ScaLAPACK/%s" % scalapack_ver, 'F'), ("FFTW-3.3.3-gompi-1.3.12.eb", "FFTW/3.3.3-gompi-1.3.12", 'F'), @@ -858,8 +884,8 @@ def test_dry_run_hierarchical(self): ("hwloc-1.6.2-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "hwloc/1.6.2", 'x'), ("OpenMPI-1.6.4-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "OpenMPI/1.6.4", 'F'), # already present and listed, so 'F' ("gompi-1.4.10.eb", "Core", "gompi/1.4.10", 'x'), - ("OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", - "OpenBLAS/0.2.6-LAPACK-3.4.2", 'x'), + ("OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb", "Compiler/GCC/4.7.2", + "OpenBLAS/0.2.6-LAPACK-3.4.2", ' '), ("FFTW-3.3.3-gompi-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", "FFTW/3.3.3", 'x'), ("ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", "ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2", 'x'), @@ -897,8 +923,8 @@ def test_dry_run_categorized(self): ("hwloc-1.6.2-GCC-4.7.2.eb", "Compiler/GCC/4.7.2/system", "hwloc/1.6.2", 'x'), ("OpenMPI-1.6.4-GCC-4.7.2.eb", "Compiler/GCC/4.7.2/mpi", "OpenMPI/1.6.4", 'F'), # already present and listed, so 'F' ("gompi-1.4.10.eb", "Core/toolchain", "gompi/1.4.10", 'x'), - ("OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", - "OpenBLAS/0.2.6-LAPACK-3.4.2", 'x'), + ("OpenBLAS-0.2.6-GCC-4.7.2-LAPACK-3.4.2.eb", "Compiler/GCC/4.7.2/numlib", + "OpenBLAS/0.2.6-LAPACK-3.4.2", ' '), ("FFTW-3.3.3-gompi-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", "FFTW/3.3.3", 'x'), ("ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", "ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2", 'x'), @@ -1588,7 +1614,7 @@ def test_hide_deps(self): outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) - self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', outtxt)) self.assertTrue(re.search('module: FFTW/3.3.3-gompi', outtxt)) self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) # zlib is not a dep at all @@ -1602,7 +1628,7 @@ def test_hide_deps(self): outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) - self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', outtxt)) self.assertFalse(re.search(r'module: FFTW/3\.3\.3-gompi', outtxt)) self.assertTrue(re.search(r'module: FFTW/\.3\.3\.3-gompi', outtxt)) self.assertFalse(re.search(r'module: ScaLAPACK/2\.0\.2-gompi', outtxt)) @@ -2439,11 +2465,11 @@ def _assert_regexs(self, regexs, txt, assert_true=True): else: self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) - def _run_mock_eb(self, args, do_build=False, raise_error=False, verbose=False, testing=True, strip=False): + def _run_mock_eb(self, args, strip=False, **kwargs): """Helper function to mock easybuild runs""" self.mock_stdout(True) self.mock_stderr(True) - self.eb_main(args, do_build=do_build, raise_error=raise_error, verbose=verbose, testing=testing) + self.eb_main(args, **kwargs) stdout_txt = self.get_stdout() stderr_txt = self.get_stderr() self.mock_stdout(False) @@ -2908,6 +2934,67 @@ def test_show_config(self): regex = re.compile(r'^include-easyblocks \(E\) = .*/testeasyblocktoinclude.py$', re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_show_config_cfg_levels(self): + """Test --show-config in relation to how configuring across multiple configuration levels interacts with it.""" + + # make sure default module syntax is used + if 'EASYBUILD_MODULE_SYNTAX' in os.environ: + del os.environ['EASYBUILD_MODULE_SYNTAX'] + + # configuring --modules-tool and --module-syntax on different levels should NOT cause problems + # cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564 + os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' + args = [ + '--module-syntax=Tcl', + '--show-config', + ] + # set init_config to False to avoid that eb_main (called by _run_mock_eb) re-initialises configuration + # this fails because $EASYBUILD_MODULES_TOOL=EnvironmentModulesC conflicts with default module syntax (Lua) + stdout, _ = self._run_mock_eb(args, raise_error=True, redo_init_config=False) + + patterns = [ + "^# Current EasyBuild configuration", + "^module-syntax\s*\(C\) = Tcl", + "^modules-tool\s*\(E\) = EnvironmentModulesC", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + def test_modules_tool_vs_syntax_check(self): + """Verify that check for modules tool vs syntax works.""" + + # make sure default module syntax is used + if 'EASYBUILD_MODULE_SYNTAX' in os.environ: + del os.environ['EASYBUILD_MODULE_SYNTAX'] + + # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem + os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' + args = ['--show-full-config'] + error_pattern = "Generating Lua module files requires Lmod as modules tool" + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + + patterns = [ + "^# Current EasyBuild configuration", + "^module-syntax\s*\(C\) = Tcl", + "^modules-tool\s*\(E\) = EnvironmentModulesC", + ] + + # EnvironmentModulesC modules tool + Tcl module syntax is fine + args.append('--module-syntax=Tcl') + stdout, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, redo_init_config=False) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + # default modules tool (Lmod) with Tcl module syntax is also fine + del os.environ['EASYBUILD_MODULES_TOOL'] + patterns[-1] = "^modules-tool\s*\(D\) = Lmod" + stdout, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, redo_init_config=False) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def test_prefix(self): """Test which configuration settings are affected by --prefix.""" txt, _ = self._run_mock_eb(['--show-full-config', '--prefix=%s' % self.test_prefix], raise_error=True) @@ -3200,22 +3287,33 @@ def test_parse_optarch(self): options.postprocess() self.assertEqual(options.options.optarch, optarch_parsed) - def test_check_style(self): - """Test --check-style.""" + def test_check_contrib_style(self): + """Test style checks performed by --check-contrib + dedicated --check-style option.""" try: - import pep8 + import pycodestyle except ImportError: - print "Skipping test_check_style, since pep8 is not available" - return + try: + import pep8 + except ImportError: + print "Skipping test_check_contrib_style, since pycodestyle or pep8 is not available" + return + regex = re.compile(r"Running style check on 2 easyconfig\(s\)(.|\n)*>> All style checks PASSed!", re.M) args = [ '--check-style', 'GCC-4.9.2.eb', 'toy-0.0.eb', ] stdout, _ = self._run_mock_eb(args, raise_error=True) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - regex = re.compile(r"Running style check on 2 easyconfig\(s\)", re.M) + # --check-contrib fails because of missing checksums, but style test passes + args[0] = '--check-contrib' + self.mock_stdout(True) + error_pattern = "One or more contribution checks FAILED" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + stdout = self.get_stdout().strip() + self.mock_stdout(False) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) # copy toy-0.0.eb test easyconfig, fiddle with it to make style check fail @@ -3229,21 +3327,50 @@ def test_check_style(self): toytxt = toytxt.replace('description = "Toy C program, 100% toy."', 'description = "%s"' % ('toy ' * 30)) write_file(toy, toytxt) + for check_type in ['contribution', 'style']: + args = [ + '--check-%s' % check_type[:7], + toy, + ] + self.mock_stdout(True) + error_pattern = "One or more %s checks FAILED!" % check_type + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + patterns = [ + "toy.eb:1:5: E223 tab before operator", + "toy.eb:1:7: E225 missing whitespace around operator", + "toy.eb:1:12: W299 trailing whitespace", + r"toy.eb:5:121: E501 line too long \(136 > 120 characters\)", + ] + for pattern in patterns: + self.assertTrue(re.search(pattern, stdout, re.M), "Pattern '%s' found in: %s" % (pattern, stdout)) + + def test_check_contrib_non_style(self): + """Test non-style checks performed by --check-contrib.""" args = [ - '--check-style', - toy, + '--check-contrib', + 'toy-0.0.eb', ] self.mock_stdout(True) - self.assertErrorRegex(EasyBuildError, "One or more style checks FAILED!", self.eb_main, args, raise_error=True) - stdout = self.get_stdout() + self.mock_stderr(True) + error_pattern = "One or more contribution checks FAILED" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + stdout = self.get_stdout().strip() + stderr = self.get_stderr().strip() self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stderr, '') + + # SHA256 checksum checks fail patterns = [ - "toy.eb:1:5: E223 tab before operator", - "toy.eb:1:7: E225 missing whitespace around operator", - "toy.eb:1:12: W299 trailing whitespace", - r"toy.eb:5:121: E501 line too long \(136 > 120 characters\)", + r"\[FAIL\] .*/toy-0.0.eb$", + r"^Checksums missing for one or more sources/patches in toy-0.0.eb: " + r"found 1 sources \+ 2 patches vs 1 checksums$", + r"^>> One or more SHA256 checksums checks FAILED!", ] for pattern in patterns: + regex = re.compile(pattern, re.M) self.assertTrue(re.search(pattern, stdout, re.M), "Pattern '%s' found in: %s" % (pattern, stdout)) def test_allow_use_as_root(self): @@ -3409,7 +3536,8 @@ def test_inject_checksums(self): self.assertEqual(ec['patches'], ['toy-0.0_fix-silly-typo-in-printf-statement.patch']) self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']}) - self.assertEqual(ec['exts_list'][0], ('bar', '0.0', { + self.assertEqual(ec['exts_list'][0], 'ls') + self.assertEqual(ec['exts_list'][1], ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': [ bar_tar_gz_sha256, @@ -3420,7 +3548,7 @@ def test_inject_checksums(self): 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', })) - self.assertEqual(ec['exts_list'][1], ('barbar', '0.0', { + self.assertEqual(ec['exts_list'][2], ('barbar', '0.0', { 'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c'], })) @@ -3436,7 +3564,7 @@ def test_inject_checksums(self): # if any checksums are present already, it doesn't matter if they're wrong (since they will be replaced) ectxt = read_file(test_ec) - for chksum in ec['checksums'] + [c for e in ec['exts_list'] for c in e[2]['checksums']]: + for chksum in ec['checksums'] + [c for e in ec['exts_list'][1:] for c in e[2]['checksums']]: ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) @@ -3533,6 +3661,43 @@ def test_force_download(self): self.assertEqual(len(toy_tar_backups), 1) self.assertTrue(os.path.basename(toy_tar_backups[0]).startswith('toy-0.0.tar.gz.bak_')) + def test_enforce_checksums(self): + """Test effect of --enforce-checksums""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + + args = [ + test_ec, + '--stop=source', + '--enforce-checksums', + ] + + # checksum is missing for patch of 'bar' extension, so --enforce-checksums should result in an error + copy_file(toy_ec, test_ec) + error_pattern = "Missing checksum for bar-0.0[^ ]*\.patch" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + + # get rid of checksums for extensions, should result in different error message + # because of missing checksum for source of 'bar' extension + regex = re.compile("^.*'checksums':.*$", re.M) + test_ec_txt = regex.sub('', read_file(test_ec)) + self.assertFalse("'checksums':" in test_ec_txt) + write_file(test_ec, test_ec_txt) + error_pattern = "Missing checksum for bar-0\.0\.tar\.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + + # wipe both exts_list and checksums, so we can check whether missing checksum for main source is caught + test_ec_txt = read_file(test_ec) + for param in ['checksums', 'exts_list']: + regex = re.compile('^%s(?:.|\n)*?\]\s*$' % param, re.M) + test_ec_txt = regex.sub('', test_ec_txt) + self.assertFalse('%s = ' % param in test_ec_txt) + + write_file(test_ec, test_ec_txt) + error_pattern = "Missing checksum for toy-0.0.tar.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/robot.py b/test/framework/robot.py index 8512e6f709..5fd6aa56b2 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -249,7 +249,7 @@ def test_resolve_dependencies(self): MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', - 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', + 'OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] @@ -331,7 +331,7 @@ def test_resolve_dependencies(self): # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second - self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name']) + self.assertEqual('OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', res[0]['full_mod_name']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name']) @@ -442,12 +442,12 @@ def test_resolve_dependencies_minimal(self): # all modules in the dep graph, in order all_mods_ordered = [ 'GCC/4.7.2', + 'OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', 'hwloc/1.6.2-GCC-4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', + 'SQLite/3.8.10.2-GCC-4.7.2', 'gompi/1.4.10', - 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', - 'SQLite/3.8.10.2-GCC-4.7.2', 'FFTW/3.3.3-gompi-1.4.10', 'goolf/1.4.10', 'bar/1.2.3-goolf-1.4.10', @@ -464,7 +464,7 @@ def test_resolve_dependencies_minimal(self): 'gompi/1.4.10', 'goolf/1.4.10', 'OpenMPI/1.6.4-GCC-4.7.2', - 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', + 'OpenBLAS/0.2.6-GCC-4.7.2-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] @@ -688,9 +688,20 @@ def test_get_toolchain_hierarchy(self): 'robot_path': test_easyconfigs, }) + goolfc_hierarchy = get_toolchain_hierarchy({'name': 'goolfc', 'version': '2.6.10'}) + self.assertEqual(goolfc_hierarchy, [ + {'name': 'GCC', 'version': '4.8.2'}, + {'name': 'golf', 'version': '2.6.10'}, + {'name': 'gcccuda', 'version': '2.6.10'}, + {'name': 'golfc', 'version': '2.6.10'}, + {'name': 'gompic', 'version': '2.6.10'}, + {'name': 'goolfc', 'version': '2.6.10'}, + ]) + goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'}) self.assertEqual(goolf_hierarchy, [ {'name': 'GCC', 'version': '4.7.2'}, + {'name': 'golf', 'version': '1.4.10'}, {'name': 'gompi', 'version': '1.4.10'}, {'name': 'goolf', 'version': '1.4.10'}, ]) @@ -992,7 +1003,7 @@ def test_robot_find_minimal_toolchain_of_dependency(self): expected_dep_versions = [ '1.6.4-GCC-4.7.2', - '0.2.6-gompi-1.4.10-LAPACK-3.4.2', + '0.2.6-GCC-4.7.2-LAPACK-3.4.2', '2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', '3.8.10.2-goolf-1.4.10', ] @@ -1115,6 +1126,31 @@ def test_check_conflicts(self): # test use of check_inter_ec_conflicts self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found") + def test_check_conflicts_wrapper_deps(self): + """Test check_conflicts when dependency 'wrappers' are involved.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + wrapper_ec_txt = '\n'.join([ + "easyblock = 'ModuleRC'", + "name = 'toy'", + "version = '0'", + "homepage = 'https://example.com'", + "description = 'Just A Wrapper'", + "toolchain = {'name': 'dummy', 'version': ''}", + "dependencies = [('toy', '0.0')]", + ]) + wrapper_ec = os.path.join(self.test_prefix, 'toy-0.eb') + write_file(wrapper_ec, wrapper_ec_txt) + + ecs, _ = parse_easyconfigs([(toy_ec, False), (wrapper_ec, False)]) + self.mock_stderr(True) + res = check_conflicts(ecs, self.modtool) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertEqual(stderr, '') + self.assertFalse(res) + def test_robot_archived_easyconfigs(self): """Test whether robot can pick up archived easyconfigs when asked.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py new file mode 100644 index 0000000000..94a683d46c --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py @@ -0,0 +1,46 @@ +## +# Copyright 2009-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy ModuleRC easyblock. + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.framework.easyblock import EasyBlock + + +class ModuleRC(EasyBlock): + """Dummy implementation of generic easyblock that generates .modulerc files.""" + + def configure_step(self): + pass + + def build_step(self): + pass + + def install_step(self): + pass + + def sanity_check_step(self): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 6f344759df..d53402c03d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -47,22 +47,25 @@ def extra_options(): def run(self): """Build toy extension.""" - super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) - EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) + if self.src: + super(Toy_Extension, self).run(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name) + EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) - if self.cfg['toy_ext_param']: - run_cmd(self.cfg['toy_ext_param']) + if self.cfg['toy_ext_param']: + run_cmd(self.cfg['toy_ext_param']) - EB_toy.install_step(self.master, name=self.name) + EB_toy.install_step(self.master, name=self.name) - return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) + return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) custom_paths = { - 'files': ['bin/%s' % self.name, 'lib/lib%s.a' % self.name], - 'dirs': [], + 'files': [], + 'dirs': ['.'], # minor hack to make sure there's always a non-empty list } + if self.src: + custom_paths['files'].extend(['bin/%s' % self.name, 'lib/lib%s.a' % self.name]) return super(Toy_Extension, self).sanity_check_step(custom_paths=custom_paths) diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 4a54cf91a0..8ba2154600 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -85,13 +85,13 @@ def test_generate_software_list(self): out, ec = run_cmd(cmd, simple=False) # make sure output is kind of what we expect it to be - regex = r"Supported Packages \(26 " + regex = r"Supported Packages \(31 " self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { 'B': '1', # bzip2 'C': '2', # CrayCCE, CUDA 'F': '1', # FFTW - 'G': '6', # GCC, GCCcore, gmvapich2, gompi, goolf, gzip + 'G': '11', # GCC, GCCcore, gcccuda, gmvapich2, golf, golfc, gompic, gompi, goolf, goolfc, gzip 'H': '1', # hwloc 'I': '8', # icc, iccifort, iccifortcuda, ictce, ifort, iimpi, imkl, impi 'M': '1', # MVAPICH2 diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6250c17943..8e6d271f7e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -42,6 +42,7 @@ from unittest import TextTestRunner from vsc.utils.fancylogger import setLogLevelDebug, logToScreen +import easybuild.tools.hooks # so we can reset cached hooks import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION @@ -1841,6 +1842,15 @@ def test_toy_build_hooks(self): "def start_hook():", " print('start hook triggered')", '', + "def parse_hook(ec):", + " print ec.name, ec.version", + # print sources value to check that raw untemplated strings are exposed in parse_hook + " print ec['sources']", + # try appending to postinstallcmd to see whether the modification is actually picked up + # (required templating to be disabled before parse_hook is called) + " ec['postinstallcmds'].append('echo toy')", + " print ec['postinstallcmds'][-1]", + '', "def pre_configure_hook(self):", " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", '', @@ -1868,6 +1878,10 @@ def test_toy_build_hooks(self): expected_output = '\n'.join([ "== Running start hook...", "start hook triggered", + "== Running parse hook for toy-0.0.eb...", + "toy 0.0", + "['%(name)s-%(version)s.tar.gz']", + "echo toy", "== Running pre-configure hook...", "pre-configure: toy.source: True", "== Running post-configure hook...", diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 5eeac6b6fa..f9ec8c56b4 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -250,7 +250,7 @@ def reset_modulepath(self, modpaths): self.modtool.set_mod_paths() def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False, - reset_env=True, raise_systemexit=False, testing=True): + reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True): """Helper method to call EasyBuild main function.""" cleanup() @@ -282,8 +282,9 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos os.chdir(self.cwd) - # make sure config is reinitialized - init_config(with_include=False) + if redo_init_config: + # make sure config is reinitialized + init_config(with_include=False) # restore environment to what it was before running main, # changes may have been made by eb_main (e.g. $TMPDIR & co)