From eeb5d17c091cdd864b2f6a11eabefeb6629c4c73 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 19 Oct 2017 16:02:47 +0200 Subject: [PATCH 001/256] Add framework support for foss-like toolchain with Spectrum MPI --- easybuild/toolchains/gsmpi.py | 39 +++++++++++++++++++++++ easybuild/toolchains/gsolf.py | 41 +++++++++++++++++++++++++ easybuild/toolchains/mpi/spectrummpi.py | 36 ++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 easybuild/toolchains/gsmpi.py create mode 100644 easybuild/toolchains/gsolf.py create mode 100644 easybuild/toolchains/mpi/spectrummpi.py diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py new file mode 100644 index 0000000000..3c09316547 --- /dev/null +++ b/easybuild/toolchains/gsmpi.py @@ -0,0 +1,39 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gompi compiler toolchain (includes GCC and SpectrumMPI). + +:author: Kenneth Hoste (Ghent University) +:author: Alan O'Cais (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.mpi.spectrummpi import SpectrumMPI + + +class Gsmpi(GccToolchain, SpectrumMPI): + """Compiler toolchain with GCC and SpectrumMPI.""" + NAME = 'gsmpi' + SUBTOOLCHAIN = GccToolchain.NAME diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py new file mode 100644 index 0000000000..44abe4d987 --- /dev/null +++ b/easybuild/toolchains/gsolf.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gsolf compiler toolchain (includes GCC, SpectrumMPI, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +:author: Kenneth Hoste (Ghent University) +:author: Alan O'Cais (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.gsmpi import Gsmpi +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + +class Gsolf(Gsmpi, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, SpectrumMPI, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'gsolf' + SUBTOOLCHAIN = Gsmpi.NAME + diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py new file mode 100644 index 0000000000..00e05c9343 --- /dev/null +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -0,0 +1,36 @@ +# # +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Support for Parastation MPI as toolchain MPI library. + +:author: Alan O'Cais (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class SpectrumMPI(OpenMPI): + """Spectrum MPI class""" + MPI_MODULE_NAME = ['SpectrumMPI'] From 7c85c6017d84cc47ee6ffe0c44d293c51ffdbfe1 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 19 Oct 2017 16:06:57 +0200 Subject: [PATCH 002/256] Update gsolf.py --- easybuild/toolchains/gsolf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 44abe4d987..8d0cf42412 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -38,4 +38,3 @@ class Gsolf(Gsmpi, OpenBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, SpectrumMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gsolf' SUBTOOLCHAIN = Gsmpi.NAME - From df29daf439a91cd4cad7a880469b4aac960ceab5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 19 Oct 2017 16:08:47 +0200 Subject: [PATCH 003/256] Update gsmpi.py --- easybuild/toolchains/gsmpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index 3c09316547..a135b74045 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for gompi compiler toolchain (includes GCC and SpectrumMPI). +EasyBuild support for gsmpi compiler toolchain (includes GCC and SpectrumMPI). :author: Kenneth Hoste (Ghent University) :author: Alan O'Cais (Juelich Supercomputing Centre) From 0101a8bccc64d271122268fa2b90fb6e903df3c8 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 19 Oct 2017 16:09:31 +0200 Subject: [PATCH 004/256] Update spectrummpi.py --- easybuild/toolchains/mpi/spectrummpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index 00e05c9343..ac717a6706 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . # # """ -Support for Parastation MPI as toolchain MPI library. +Support for Spectrum MPI as toolchain MPI library. :author: Alan O'Cais (Juelich Supercomputing Centre) """ From ce1b2740c1d718ed95be0a0d9a091d4d9782977a Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 23 Oct 2017 10:08:43 +0800 Subject: [PATCH 005/256] review-pr before submitting new-pr --- easybuild/framework/easyconfig/tools.py | 25 +++++++++++++++++++++++++ easybuild/main.py | 6 +++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 9e75e29319..ee26c8dfa9 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -499,6 +499,31 @@ def review_pr(pr, colored=True, branch='develop'): return '\n'.join(lines) +def review_new_pr(paths, colored=True, branch='develop'): + """ + Print multi-diff overview between easyconfigs in new PR and specified branch. + :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) + :param colored: boolean indicating whether a colored multi-diff should be generated + :param branch: easybuild-easyconfigs branch to compare with + """ + tmpdir = tempfile.mkdtemp() + + download_repo_path = download_repo(branch=branch, path=tmpdir) + repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') + pr_files = [path for path in paths['easyconfigs']] + + lines = [] + ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) + for ec in ecs: + files = find_related_easyconfigs(repo_path, ec['ec']) + _log.debug("File in new PR %s has these related easyconfigs: %s" % (ec['spec'], files)) + if files: + lines.append(multidiff(ec['spec'], files, colored=colored)) + else: + lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) + + return '\n'.join(lines) + def dump_env_script(easyconfigs): """ diff --git a/easybuild/main.py b/easybuild/main.py index ae756f0477..ee9de84882 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available +from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, review_new_pr, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software @@ -403,6 +403,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # creating/updating PRs if new_update_pr: if options.new_pr: + + if options.extended_dry_run: + print review_new_pr(categorized_paths, colored=use_color(options.color)) + new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) else: From 792cc087be1c3d9cb96b85cc81c5f0fc68532941 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 24 Oct 2017 08:58:45 +0800 Subject: [PATCH 006/256] use det_easyconfig_paths --- easybuild/framework/easyconfig/tools.py | 4 ++-- easybuild/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ee26c8dfa9..1bb1ea7f39 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -502,7 +502,7 @@ def review_pr(pr, colored=True, branch='develop'): def review_new_pr(paths, colored=True, branch='develop'): """ Print multi-diff overview between easyconfigs in new PR and specified branch. - :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) + :param paths: path tuples (path, generated) :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with """ @@ -510,7 +510,7 @@ def review_new_pr(paths, colored=True, branch='develop'): download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') - pr_files = [path for path in paths['easyconfigs']] + pr_files = [path[0] for path in paths] lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) diff --git a/easybuild/main.py b/easybuild/main.py index ee9de84882..f04a40a431 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -405,7 +405,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.new_pr: if options.extended_dry_run: - print review_new_pr(categorized_paths, colored=use_color(options.color)) + print review_new_pr(paths, colored=use_color(options.color)) new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) From 29aea71d68c75c7c82413b931522dace95a04d3e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 24 Oct 2017 09:06:54 +0800 Subject: [PATCH 007/256] new option --preview-pr --- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/main.py | 14 +++++++------- easybuild/tools/options.py | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 1bb1ea7f39..2552ed0a14 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -499,7 +499,7 @@ def review_pr(pr, colored=True, branch='develop'): return '\n'.join(lines) -def review_new_pr(paths, colored=True, branch='develop'): +def preview_pr(paths, colored=True, branch='develop'): """ Print multi-diff overview between easyconfigs in new PR and specified branch. :param paths: path tuples (path, generated) diff --git a/easybuild/main.py b/easybuild/main.py index f04a40a431..15f25023d2 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, review_new_pr, skip_available +from easybuild.framework.easyconfig.tools import parse_easyconfigs, preview_pr, review_pr, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software @@ -375,7 +375,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short - new_update_pr = options.new_pr or options.update_pr + new_update_pr = options.new_pr or options.update_pr or options.preview_pr # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_pr or options.inject_checksums): @@ -402,11 +402,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # creating/updating PRs if new_update_pr: - if options.new_pr: - - if options.extended_dry_run: - print review_new_pr(paths, colored=use_color(options.color)) - + if options.preview_pr: + print preview_pr(paths, colored=use_color(options.color)) + cleanup(logfile, eb_tmpdir, testing, silent=True) + sys.exit(0) + elif options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) else: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ccefecba19..d4385ccce5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -569,6 +569,7 @@ def github_options(self): 'pr-target-branch': ("Target branch for new PRs", str, 'store', 'develop'), 'pr-target-repo': ("Target repository for new/updating PRs", str, 'store', GITHUB_EASYCONFIGS_REPO), 'pr-title': ("Title for new pull request created with --new-pr", str, 'store', None), + 'preview-pr': ("Preview a new pull request", None, 'store_true', False), 'review-pr': ("Review specified pull request", int, 'store', None, {'metavar': 'PR#'}), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), From 468f15480b3294e1c168284cf1cebda1a27d56f5 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 24 Oct 2017 19:29:41 +0800 Subject: [PATCH 008/256] reduce code duplication --- easybuild/framework/easyconfig/tools.py | 45 ++++++++----------------- easybuild/main.py | 10 +++--- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2552ed0a14..2cfbe7cf7b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -473,11 +473,11 @@ def find_related_easyconfigs(path, ec): return sorted(res) - -def review_pr(pr, colored=True, branch='develop'): +def review_pr(paths=None, pr=None, colored=True, branch='develop'): """ - Print multi-diff overview between easyconfigs in specified PR and specified branch. + Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review + :param paths: path tuples (path, generated) of easyconfigs to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with """ @@ -485,38 +485,22 @@ def review_pr(pr, colored=True, branch='develop'): download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') - pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] - + + if pr: + pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] + elif paths: + pr_files = [path[0] for path in paths] + else: + return "no easyconfigs or pr specified" + lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: files = find_related_easyconfigs(repo_path, ec['ec']) - _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec['spec'], files)) - if files: - lines.append(multidiff(ec['spec'], files, colored=colored)) + if pr: + _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec['spec'], files)) else: - lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) - - return '\n'.join(lines) - -def preview_pr(paths, colored=True, branch='develop'): - """ - Print multi-diff overview between easyconfigs in new PR and specified branch. - :param paths: path tuples (path, generated) - :param colored: boolean indicating whether a colored multi-diff should be generated - :param branch: easybuild-easyconfigs branch to compare with - """ - tmpdir = tempfile.mkdtemp() - - download_repo_path = download_repo(branch=branch, path=tmpdir) - repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') - pr_files = [path[0] for path in paths] - - lines = [] - ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) - for ec in ecs: - files = find_related_easyconfigs(repo_path, ec['ec']) - _log.debug("File in new PR %s has these related easyconfigs: %s" % (ec['spec'], files)) + _log.debug("File in new PR %s has these related easyconfigs: %s" % (ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: @@ -524,7 +508,6 @@ def preview_pr(paths, colored=True, branch='develop'): return '\n'.join(lines) - def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. diff --git a/easybuild/main.py b/easybuild/main.py index 15f25023d2..2b3807d650 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, preview_pr, review_pr, skip_available +from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software @@ -276,7 +276,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): merge_pr(options.merge_pr) elif options.review_pr: - print review_pr(options.review_pr, colored=use_color(options.color)) + print review_pr(pr=options.review_pr, colored=use_color(options.color)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' @@ -323,7 +323,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified - no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query] + no_ec_opts = [options.aggregate_regtest, options.new_pr, options.preview_pr, options.regtest, options.update_pr, search_query] # determine paths to easyconfigs paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -403,9 +403,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # creating/updating PRs if new_update_pr: if options.preview_pr: - print preview_pr(paths, colored=use_color(options.color)) - cleanup(logfile, eb_tmpdir, testing, silent=True) - sys.exit(0) + print review_pr(paths=paths, colored=use_color(options.color)) elif options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) From e2b0a976eee9f6c8c4d2a7ac480b2b5096459b89 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 25 Oct 2017 09:24:37 +0800 Subject: [PATCH 009/256] raise error if no PR # or easyconfig path is specified --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2cfbe7cf7b..470b913fd2 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -491,7 +491,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): elif paths: pr_files = [path[0] for path in paths] else: - return "no easyconfigs or pr specified" + raise EasyBuildError("No PR # or easyconfig path specified") lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) From 77ddd190510129aa96877d2a49d8fc671dcd3a36 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 25 Oct 2017 09:26:17 +0800 Subject: [PATCH 010/256] avoid code repetition --- easybuild/framework/easyconfig/tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 470b913fd2..908f122c40 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -498,9 +498,10 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): for ec in ecs: files = find_related_easyconfigs(repo_path, ec['ec']) if pr: - _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec['spec'], files)) + pr_msg = "PR#%s" % pr else: - _log.debug("File in new PR %s has these related easyconfigs: %s" % (ec['spec'], files)) + pr_msg = "new PR" + _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: From 79c991330cda982b8ab1a1e0d53ce441abfcb0c0 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 25 Oct 2017 09:28:45 +0800 Subject: [PATCH 011/256] fix style --- easybuild/framework/easyconfig/tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 908f122c40..52192345ac 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -473,6 +473,7 @@ def find_related_easyconfigs(path, ec): return sorted(res) + def review_pr(paths=None, pr=None, colored=True, branch='develop'): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. @@ -485,14 +486,14 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') - + if pr: pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] elif paths: pr_files = [path[0] for path in paths] else: raise EasyBuildError("No PR # or easyconfig path specified") - + lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: @@ -509,6 +510,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): return '\n'.join(lines) + def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. From 3ba5bf99750841d261862612a681aa23f6adc020 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 25 Oct 2017 09:55:53 +0800 Subject: [PATCH 012/256] redefine no_ec_opts and new_update_preview_pr --- easybuild/main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 2b3807d650..6c8502509c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -323,7 +323,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified - no_ec_opts = [options.aggregate_regtest, options.new_pr, options.preview_pr, options.regtest, options.update_pr, search_query] + no_ec_opts = [options.aggregate_regtest, options.regtest, search_query] # determine paths to easyconfigs paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -375,10 +375,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short - new_update_pr = options.new_pr or options.update_pr or options.preview_pr + new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping - if not (forced or dry_run_mode or options.extended_dry_run or new_update_pr or options.inject_checksums): + if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: @@ -389,24 +389,24 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled - if options.robot and (not dry_run_mode or new_update_pr): + if options.robot and (not dry_run_mode or new_update_preview_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif new_update_pr: + elif new_update_preview_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs - if new_update_pr: - if options.preview_pr: - print review_pr(paths=paths, colored=use_color(options.color)) - elif options.new_pr: + if new_update_preview_pr: + if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) + elif options.preview_pr: + print review_pr(paths=paths, colored=use_color(options.color)) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) @@ -430,7 +430,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test - if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]): + if any(no_ec_opts + [new_update_preview_pr, options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) From a0867751d7f6a0d0b17aa1f22c6851c6124a1aea Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 25 Oct 2017 10:06:04 +0800 Subject: [PATCH 013/256] use original determined paths --- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/main.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 52192345ac..74e5d0677c 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -490,7 +490,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): if pr: pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] elif paths: - pr_files = [path[0] for path in paths] + pr_files = paths else: raise EasyBuildError("No PR # or easyconfig path specified") diff --git a/easybuild/main.py b/easybuild/main.py index 6c8502509c..d9c1564b03 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -326,10 +326,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): no_ec_opts = [options.aggregate_regtest, options.regtest, search_query] # determine paths to easyconfigs - paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if paths: + determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) + if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated - paths = [(p, False) for p in paths] + paths = [(p, False) for p in determined_paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided @@ -406,7 +406,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.preview_pr: - print review_pr(paths=paths, colored=use_color(options.color)) + print review_pr(paths=determined_paths, colored=use_color(options.color)) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) From 0354df1b9b372f608db20e091a2a8759867e601e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 16:01:45 +0200 Subject: [PATCH 014/256] fix use of --inject-checksums on bundles that do not have a 'sources' spec --- easybuild/framework/easyblock.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bc1c9d60db..e91bcc984b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2957,16 +2957,13 @@ def inject_checksums(ecs, checksum_type): raw[key] = res.group(0).strip() + '\n' ectxt = regex.sub(placeholder, ectxt) - # this should not happen... - if 'sources' not in raw: - raise EasyBuildError("Failed to extract raw lines for 'sources' parameter from easyconfig file!") - # inject combination of source_urls/sources/patches/checksums into easyconfig # by replacing first occurence of placeholder that was put in place + sources_raw = raw.get('sources', '') source_urls_raw = raw.get('source_urls', '') patches_raw = raw.get('patches', '') regex = re.compile(placeholder + '\n', re.M) - ectxt = regex.sub(source_urls_raw + raw['sources'] + patches_raw + checksums_txt + '\n', ectxt, count=1) + ectxt = regex.sub(source_urls_raw + sources_raw + patches_raw + checksums_txt + '\n', ectxt, count=1) # get rid of potential remaining placeholders ectxt = regex.sub('', ectxt) From 7cc646297fb917d5925eac4dbf375953160a8842 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 16:02:26 +0200 Subject: [PATCH 015/256] inject single checksum for extensions on a single line (& omit filename) --- easybuild/framework/easyblock.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e91bcc984b..b9a42aa797 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3004,10 +3004,13 @@ def inject_checksums(ecs, checksum_type): # if any checksums were collected, inject them for this extension if ext_checksums: - exts_list_lines.append("%s'checksums': [" % (INDENT_4SPACES * 2)) - for fn, checksum in ext_checksums: - exts_list_lines.append("%s'%s', # %s" % (INDENT_4SPACES * 3, checksum, fn)) - exts_list_lines.append("%s]," % (INDENT_4SPACES * 2)) + if len(ext_checksums) == 1: + exts_list_lines.append("%s'checksums': ['%s']," % (INDENT_4SPACES * 2, checksum)) + else: + exts_list_lines.append("%s'checksums': [" % (INDENT_4SPACES * 2)) + for fn, checksum in ext_checksums: + exts_list_lines.append("%s'%s', # %s" % (INDENT_4SPACES * 3, checksum, fn)) + exts_list_lines.append("%s]," % (INDENT_4SPACES * 2)) if ext_options or ext_checksums: exts_list_lines.append("%s})," % INDENT_4SPACES) From d203b74a63d70cd0b78e1bc218ed3fb7c019b63f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 16:13:02 +0200 Subject: [PATCH 016/256] avoid hardcoding name/version of extension that matches parent software name/version under --inject-checksums, use name/version parameters instead --- easybuild/framework/easyblock.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b9a42aa797..6101dd331a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2974,10 +2974,21 @@ def inject_checksums(ecs, checksum_type): exts_list_lines = ['exts_list = ['] for ext in app.exts: + if ext['name'] == app.name: + ext_name = 'name' + else: + ext_name = "'%s'" % ext['name'] + # for some extensions, only a name if specified (so no sources/patches) if ext.keys() == ['name']: - exts_list_lines.append("%s'%s'," % (INDENT_4SPACES, ext['name'])) + exts_list_lines.append("%s%s," % (INDENT_4SPACES, ext_name)) else: + + if ext['version'] == app.version: + ext_version = 'version' + else: + ext_version = "'%s'" % ext['version'] + ext_options = ext.get('options', {}) # compute checksums for extension sources & patches @@ -2993,7 +3004,7 @@ def inject_checksums(ecs, checksum_type): print_msg(" * %s: %s" % (patch_fn, checksum), log=_log) ext_checksums.append((patch_fn, checksum)) - exts_list_lines.append("%s('%s', '%s'," % (INDENT_4SPACES, ext['name'], ext['version'])) + exts_list_lines.append("%s(%s, %s," % (INDENT_4SPACES, ext_name, ext_version)) if ext_options or ext_checksums: exts_list_lines[-1] += ' {' From 640062f4db2a09e5297d9197df63975710fdd8dd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 16:38:23 +0200 Subject: [PATCH 017/256] enhance test for --inject-checksums to also cover fixed bugs --- .../test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb | 1 + test/framework/options.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index d2e54fd6ae..799bb1699b 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -31,6 +31,7 @@ exts_list = [ 'unknowneasyconfigparameterthatshouldbeignored': 'foo', }), ('barbar', '0.0'), + (name, version), ] sanity_check_paths = { diff --git a/test/framework/options.py b/test/framework/options.py index aabad50967..91c580c3fb 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3361,12 +3361,19 @@ def test_inject_checksums(self): warning_msg = "WARNING: Found existing checksums in test.eb, overwriting them (due to use of --force)..." self.assertEqual(stderr, warning_msg) - # make sure checksums are only there once... ec_txt = read_file(test_ec) + + # some checks on 'raw' easyconfig contents + # single-line checksum for barbar extension since there's only one + self.assertTrue("'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c']," in ec_txt) + # name/version of toy should NOT be hardcoded in exts_list, 'name'/'version' parameters should be used + self.assertTrue(' (name, version, {' in ec_txt) + + # make sure checksums are only there once... # exactly one definition of 'checksums' easyconfig parameter self.assertEqual(re.findall('^checksums', ec_txt, re.M), ['checksums']) - # exactly two checksum specs for extensions, one list of checksums for each extension - self.assertEqual(re.findall("[ ]*'checksums'", ec_txt, re.M), [" 'checksums'", " 'checksums'"]) + # exactly three checksum specs for extensions, one list of checksums for each extension + self.assertEqual(re.findall("[ ]*'checksums'", ec_txt, re.M), [" 'checksums'"] * 3) # no parse errors for updated easyconfig file... ec = EasyConfigParser(test_ec).get_config_dict() From 38afd586e1ba65a7698a9365f5143c8576f5ff90 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 21:29:40 +0200 Subject: [PATCH 018/256] fixes for using toy as extension for toy (inception!) --- .../test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb | 5 ++++- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index 799bb1699b..326519c88c 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -31,7 +31,10 @@ exts_list = [ 'unknowneasyconfigparameterthatshouldbeignored': 'foo', }), ('barbar', '0.0'), - (name, version), + (name, version, { + 'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []}, + 'exts_filter': ("ls -l lib/libtoy.a", ''), + }), ] sanity_check_paths = { diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index bec4d1a803..6d39b41e1b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -32,7 +32,7 @@ import platform import shutil -from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir @@ -40,7 +40,7 @@ from easybuild.tools.run import run_cmd -class EB_toy(EasyBlock): +class EB_toy(ExtensionEasyBlock): """Support for building/installing toy.""" def __init__(self, *args, **kwargs): @@ -99,6 +99,13 @@ def install_step(self, name=None): f.write(name.upper()) f.close() + def run(self): + """Install toy as extension.""" + super(EB_toy, self).run(unpack_src=True) + self.configure_step() + self.build_step() + self.install_step() + def make_module_extra(self): """Extra stuff for toy module""" txt = super(EB_toy, self).make_module_extra() From 51624a6f921b0d59ffa20d271f5b4d56c17cd97e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Oct 2017 22:15:03 +0200 Subject: [PATCH 019/256] don't blindly change into install dir in sanity_check_step for extensions (to avoid trouble under --module-only) --- easybuild/framework/extension.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 2f91c2da5c..9af9b1b8df 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -134,7 +134,8 @@ def sanity_check_step(self): Sanity check to run after installing extension """ - change_dir(self.installdir) + if os.path.isdir(self.installdir): + change_dir(self.installdir) # disabling templating is required here to support legacy string templates like name/version self.cfg.enable_templating = False From f2c8bfd290873662afe734f62d93c4338270d689 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 26 Oct 2017 08:39:54 +0800 Subject: [PATCH 020/256] fix indentation --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 74e5d0677c..317f4ae673 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -502,7 +502,7 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): pr_msg = "PR#%s" % pr else: pr_msg = "new PR" - _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) + _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: From bde283d2eade87627396303e39e0dfb4daa9768e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 26 Oct 2017 08:44:11 +0800 Subject: [PATCH 021/256] shorten long line --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index d9c1564b03..f9b35841cc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -430,7 +430,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test - if any(no_ec_opts + [new_update_preview_pr, options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]): + stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] + if no_ec_opts or new_update_preview_pr or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) From 25e9b5614924dc15f0ddb61476c63fa5aab66bc2 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 26 Oct 2017 10:06:13 +0800 Subject: [PATCH 022/256] if no_ec_opts should be if any(no_ec_opts) --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index f9b35841cc..bdab3ba455 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -431,7 +431,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] - if no_ec_opts or new_update_preview_pr or any(stop_options): + if any(no_ec_opts) or new_update_preview_pr or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) From ca98281979367d86df0e86d803604e349b3b94ef Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 30 Oct 2017 09:54:55 +0800 Subject: [PATCH 023/256] add test --- test/framework/options.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index aabad50967..29f62f9497 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2185,6 +2185,24 @@ def test_cleanup_tmpdir(self): tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') self.assertTrue(os.path.exists(os.path.join(tweaked_dir, 'toy-1.0.eb'))) + def test_preview_pr(self): + """Test --preview-pr.""" + + self.mock_stdout(True) + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') + args = [ + '--color=never', + '--preview-pr', + eb_file, + ] + self.eb_main(args, raise_error=True) + txt = self.get_stdout() + self.mock_stdout(False) + regex = re.compile(r"^Comparing bzip2-1.0.6\S* with bzip2-1.0.6") + self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) + def test_review_pr(self): """Test --review-pr.""" if self.github_token is None: From 4d50c55b3b829e9c80714a78f36e40b9db8f22f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Oct 2017 09:40:41 +0100 Subject: [PATCH 024/256] flesh out find_extension function, hoist dict with extract commands into a constant --- easybuild/tools/filetools.py | 79 ++++++++++++++++++++---------------- test/framework/filetools.py | 28 +++++++++++++ 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a7c7f83185..4374b4be4d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -120,6 +120,32 @@ } CHECKSUM_TYPES = sorted(CHECKSUM_FUNCTIONS.keys()) +EXTRACT_CMDS = { + # gzipped or gzipped tarball + '.gtgz': "tar xzf %(filepath)s", + '.gz': "gunzip -c %(filepath)s > %(target)s", + '.tar.gz': "tar xzf %(filepath)s", + '.tgz': "tar xzf %(filepath)s", + # bzipped or bzipped tarball + '.bz2': "bunzip2 -c %(filepath)s > %(target)s", + '.tar.bz2': "tar xjf %(filepath)s", + '.tb2': "tar xjf %(filepath)s", + '.tbz': "tar xjf %(filepath)s", + '.tbz2': "tar xjf %(filepath)s", + # xzipped or xzipped tarball + '.tar.xz': "unxz %(filepath)s --stdout | tar x", + '.txz': "unxz %(filepath)s --stdout | tar x", + '.xz': "unxz %(filepath)s", + # tarball + '.tar': "tar xf %(filepath)s", + # zip file + '.zip': "unzip -qq %(filepath)s", + # iso file + '.iso': "7z x %(filepath)s", + # tar.Z: using compress (LZW) + '.tar.z': "tar xZf %(filepath)s", +} + class ZlibChecksum(object): """ @@ -715,49 +741,32 @@ def get_local_dirs_purged(): return new_dir -def extract_cmd(filepath, overwrite=False): - """ - Determines the file type of file at filepath, returns extract cmd based on file suffix - """ - filename = os.path.basename(filepath) - - extract_cmds = { - # gzipped or gzipped tarball - '.gtgz': "tar xzf %(filepath)s", - '.gz': "gunzip -c %(filepath)s > %(target)s", - '.tar.gz': "tar xzf %(filepath)s", - '.tgz': "tar xzf %(filepath)s", - # bzipped or bzipped tarball - '.bz2': "bunzip2 -c %(filepath)s > %(target)s", - '.tar.bz2': "tar xjf %(filepath)s", - '.tb2': "tar xjf %(filepath)s", - '.tbz': "tar xjf %(filepath)s", - '.tbz2': "tar xjf %(filepath)s", - # xzipped or xzipped tarball - '.tar.xz': "unxz %(filepath)s --stdout | tar x", - '.txz': "unxz %(filepath)s --stdout | tar x", - '.xz': "unxz %(filepath)s", - # tarball - '.tar': "tar xf %(filepath)s", - # zip file - '.zip': "unzip -qq -o %(filepath)s" if overwrite else "unzip -qq %(filepath)s", - # iso file - '.iso': "7z x %(filepath)s", - # tar.Z: using compress (LZW) - '.tar.z': "tar xZf %(filepath)s", - } - - suffixes = sorted(extract_cmds.keys(), key=len, reverse=True) - pat = r'(?P%s)$' % '|'.join([ext.replace('.', '\\.') for ext in suffixes]) +def find_extension(filename): + """Find best match for filename extension.""" + # sort by length, so longest file extensions get preference + suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True) + pat = r'(?P%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes]) res = re.search(pat, filename, flags=re.IGNORECASE) if res: ext = res.group('ext') else: raise EasyBuildError('Unknown file type for file %s', filename) + return ext + + +def extract_cmd(filepath, overwrite=False): + """ + Determines the file type of file at filepath, returns extract cmd based on file suffix + """ + filename = os.path.basename(filepath) + ext = find_extension(filename) target = filename.rstrip(ext) - cmd_tmpl = extract_cmds[ext.lower()] + cmd_tmpl = EXTRACT_CMDS[ext.lower()] + if overwrite: + if 'unzip -qq' in cmd_tmpl: + cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o') return cmd_tmpl % {'filepath': filepath, 'target': target} diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 910796859c..8a13afb1ae 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -89,6 +89,34 @@ def test_extract_cmd(self): self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + def test_find_extension(self): + """Test find_extension function.""" + tests = [ + ('test.zip', '.zip'), + ('/some/path/test.tar', '.tar'), + ('test.tar.gz', '.tar.gz'), + ('test.TAR.GZ', '.TAR.GZ'), + ('test.tgz', '.tgz'), + ('test.gtgz', '.gtgz'), + ('test.bz2', '.bz2'), + ('/some/path/test.bz2', '.bz2'), + ('test.tbz', '.tbz'), + ('test.tbz2', '.tbz2'), + ('test.tb2', '.tb2'), + ('test.tar.bz2', '.tar.bz2'), + ('test.gz', '.gz'), + ('untar.gz', '.gz'), + ("/some/path/test.gz", '.gz'), + ('test.xz', '.xz'), + ('test.tar.xz', '.tar.xz'), + ('test.txz', '.txz'), + ('test.iso', '.iso'), + ('test.tar.Z', '.tar.Z'), + ] + for (fn, expected_ext) in tests: + cmd = ft.find_extension(fn) + self.assertEqual(expected_ext, cmd) + def test_convert_name(self): """Test convert_name function.""" name = ft.convert_name("test+test-test.mpi") From 7d94eaecbfe4abbcd48646bf78ec7c4c357b524f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Oct 2017 10:47:05 +0100 Subject: [PATCH 025/256] --new-pr & co don't strictly require easyconfigs to be specified (e.g. for patches, removing easyconfigs, ...) --- easybuild/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index bdab3ba455..a24fe9f201 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -323,7 +323,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified - no_ec_opts = [options.aggregate_regtest, options.regtest, search_query] + new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr + no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -334,7 +335,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) - elif not any(no_ec_opts): + elif no_ec_opts: + paths = determined_paths + else: print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) @@ -375,7 +378,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short - new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): @@ -431,7 +433,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] - if any(no_ec_opts) or new_update_preview_pr or any(stop_options): + if any(no_ec_opts) or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) From 2e88c9f84cdc6d0ba71f2b9116f2c96e95f57087 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Oct 2017 11:00:03 +0100 Subject: [PATCH 026/256] add missing 'any' --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index a24fe9f201..3529e4ceec 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -335,7 +335,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) - elif no_ec_opts: + elif any(no_ec_opts): paths = determined_paths else: print_error(("Please provide one or multiple easyconfig files, or use software build " From f64f5220329f70f3875ceb7ea52c6b0e1182d965 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Nov 2017 14:07:17 +0100 Subject: [PATCH 027/256] add support for using self.start_dir rather than using self.cfg['start_dir'] --- easybuild/framework/easyblock.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6101dd331a..194cb6096b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1390,6 +1390,11 @@ def skip_extensions(self): # MISCELLANEOUS UTILITY FUNCTIONS # + @property + def start_dir(self): + """Start directory in build directory""" + return self.cfg['start_dir'] + def guess_start_dir(self): """ Return the directory where to start the whole configure/make/make install cycle from @@ -1401,8 +1406,8 @@ def guess_start_dir(self): start_dir = '' # do not use the specified 'start_dir' when running as --module-only as # the directory will not exist (extract_step is skipped) - if self.cfg['start_dir'] and not build_option('module_only'): - start_dir = self.cfg['start_dir'] + if self.start_dir and not build_option('module_only'): + start_dir = self.start_dir if not os.path.isabs(start_dir): if len(self.src) > 0 and not self.skip and self.src[0]['finalpath']: @@ -1428,8 +1433,8 @@ def guess_start_dir(self): self.log.info("Using %s as start dir", self.cfg['start_dir']) - change_dir(self.cfg['start_dir']) - self.log.debug("Changed to real build directory %s (start_dir)", self.cfg['start_dir']) + change_dir(self.start_dir) + self.log.debug("Changed to real build directory %s (start_dir)", self.start_dir) def handle_iterate_opts(self): """Handle options relevant during iterated part of build/install procedure.""" From e0c52a8353bba14ff5cb6d9ca3aff1f4dbebc742 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Nov 2017 10:07:09 +0100 Subject: [PATCH 028/256] fix error message when 'gv' Python package is not available --- easybuild/framework/easyconfig/tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 317f4ae673..bb63cc2758 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -75,7 +75,8 @@ sys.path.append('..') sys.path.append('/usr/lib/graphviz/python/') sys.path.append('/usr/lib64/graphviz/python/') - # https://pypi.python.org/pypi/pygraphviz + # Python bindings to Graphviz (http://www.graphviz.org/), + # see https://pypi.python.org/pypi/graphviz-python # graphviz-python (yum) or python-pygraphviz (apt-get) # or brew install graphviz --with-bindings (OS X) import gv @@ -225,7 +226,7 @@ def _dep_graph_dump(dgr, filename): _dep_graph_gv(dottxt, filename) -@only_if_module_is_available('gv', pkgname='graphviz') +@only_if_module_is_available('gv', pkgname='graphviz-python') def _dep_graph_gv(dottxt, filename): """Render dependency graph to file using graphviz.""" # try and render graph in specified file format From a389a3b55959e05579631e161add7263c4cd03dc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Nov 2017 09:30:09 +0100 Subject: [PATCH 029/256] first implementation of support for pre- and post-step hooks --- easybuild/framework/easyblock.py | 16 ++++++++++++--- easybuild/tools/config.py | 9 ++++---- easybuild/tools/filetools.py | 35 ++++++++++++++++++++++++++++++++ easybuild/tools/options.py | 7 ++++++- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6101dd331a..6b3d00e4f0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,9 +70,8 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file -from easybuild.tools.filetools import remove_file, rmtree2, write_file -from easybuild.tools.filetools import verify_checksum, weld_paths +from easybuild.tools.filetools import encode_class_name, extract_file, find_hook, is_alt_pypi_url, mkdir, move_logs +from easybuild.tools.filetools import read_file, remove_file, rmtree2, write_file, verify_checksum, weld_paths from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -2434,6 +2433,12 @@ def run_step(self, step, step_methods): """ self.log.info("Starting %s step", step) self.update_config_template_run_step() + + pre_hook = find_hook(step, pre_hook=True) + if pre_hook: + self.log.info("Found pre-%s hook, so running it...", step) + pre_hook(self) + for step_method in step_methods: self.log.info("Running method %s part of step %s" % ('_'.join(step_method.func_code.co_names), step)) @@ -2457,6 +2462,11 @@ def run_step(self, step, step_methods): # and returns the actual method, so use () to execute it step_method(self)() + post_hook = find_hook(step, pre_hook=False) + if post_hook: + self.log.info("Found post-%s hook, so running it...", step) + post_hook(self) + if self.cfg['stop'] == step: self.log.info("Stopping after %s step.", step) raise StopException(step) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index a055cd8063..05f4b82370 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -121,10 +121,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'force_download', 'from_pr', 'git_working_dirs_path', - 'pr_branch_name', - 'pr_target_account', - 'pr_target_branch', - 'pr_target_repo', + 'hooks', 'github_user', 'github_org', 'group', @@ -142,6 +139,10 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'optarch', 'package_tool_options', 'parallel', + 'pr_branch_name', + 'pr_target_account', + 'pr_target_branch', + 'pr_target_repo', 'rpath_filter', 'regtest_output_dir', 'skip', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4374b4be4d..618ff7bc34 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -42,6 +42,7 @@ import fileinput import glob import hashlib +import imp import os import re import shutil @@ -1651,3 +1652,37 @@ def diff_files(path1, path2): file1_lines = ['%s\n' % l for l in read_file(path1).split('\n')] file2_lines = ['%s\n' % l for l in read_file(path2).split('\n')] return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2)) + + +def find_hook(step_name, pre_hook=True): + """ + Find pre- or post-hook for specified step. + + :param step_name: name of the step that hook relates to + :param pre_hook: True to search for pre-step hook, False to search for post-step hook + """ + res = None + hooks_path = build_option('hooks') + if hooks_path: + (hooks_dir, hooks_filename) = os.path.split(hooks_path) + (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) + if hooks_file_ext == '.py': + _log.info("Importing hooks implementation from %s...", hooks_path) + (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir]) + try: + imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr) + found_hooks = [attr for attr in dir(imported_hooks) if attr.endswith('_hook')] + _log.debug("Found hooks: %s", found_hooks) + except ImportError as err: + raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) + + hook_name = ('post_', 'pre_')[pre_hook] + step_name + '_hook' + if hook_name in found_hooks: + res = getattr(imported_hooks, hook_name) + print "Found %s: %s" % (hook_name, res) + else: + raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)") + else: + _log.info("No location for hooks implementation provided, no hooks defined") + + return res diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d4385ccce5..a366c81fdc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -70,7 +70,7 @@ from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir, resolve_path from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, HAVE_GITHUB_API, HAVE_KEYRING from easybuild.tools.github import fetch_github_token from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains @@ -433,6 +433,7 @@ def config_options(self): 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", 'strlist', 'store', None), + 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'include-easyblocks': ("Location(s) of extra or customized easyblocks", 'strlist', 'store', []), @@ -779,6 +780,10 @@ def postprocess(self): # handle configuration options that affect other configuration options self._postprocess_config() + # make sure absolute path is used for location of hooks + if self.options.hooks: + self.options.hooks = resolve_path(self.options.hooks) + # show current configuration and exit, if requested if self.options.show_config or self.options.show_full_config: self.show_config() From 5523e8516d7923b3fa0ceb8888c90ffb345caf12 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Nov 2017 09:56:41 +0100 Subject: [PATCH 030/256] only load implementation of hooks once --- easybuild/framework/easyblock.py | 13 ++++++---- easybuild/main.py | 18 +++++++++---- easybuild/tools/filetools.py | 44 +++++++++++++++++++++----------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6b3d00e4f0..8386bab8a1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -135,7 +135,7 @@ def extra_options(extra=None): # # INIT # - def __init__(self, ec): + def __init__(self, ec, hooks=None): """ Initialize the EasyBlock instance. :param ec: a parsed easyconfig file (EasyConfig instance) @@ -144,6 +144,9 @@ def __init__(self, ec): # keep track of original working directory, so we can go back there self.orig_workdir = os.getcwd() + # list of pre- and post-step hooks + self.hooks = hooks + # list of patch/source files, along with checksums self.patches = [] self.src = [] @@ -2434,7 +2437,7 @@ def run_step(self, step, step_methods): self.log.info("Starting %s step", step) self.update_config_template_run_step() - pre_hook = find_hook(step, pre_hook=True) + pre_hook = find_hook(step, self.hooks, pre_hook=True) if pre_hook: self.log.info("Found pre-%s hook, so running it...", step) pre_hook(self) @@ -2462,7 +2465,7 @@ def run_step(self, step, step_methods): # and returns the actual method, so use () to execute it step_method(self)() - post_hook = find_hook(step, pre_hook=False) + post_hook = find_hook(step, self.hooks, pre_hook=False) if post_hook: self.log.info("Found post-%s hook, so running it...", step) post_hook(self) @@ -2611,7 +2614,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env): +def build_and_install_one(ecdict, init_env, hooks=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -2649,7 +2652,7 @@ def build_and_install_one(ecdict, init_env): try: app_class = get_easyblock_class(easyblock, name=name) - app = app_class(ecdict['ec']) + app = app_class(ecdict['ec'], hooks=hooks) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), diff --git a/easybuild/main.py b/easybuild/main.py index 3529e4ceec..54814f7982 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -58,7 +58,7 @@ from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, load_hooks, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.modules import modules_tool @@ -104,8 +104,15 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True): - """Build and install software for all provided parsed easyconfig files.""" +def build_and_install_software(ecs, init_session_state, hooks=None, exit_on_failure=True): + """ + Build and install software for all provided parsed easyconfig files. + + :param ecs: easyconfig files to install software with + :param init_session_state: initial session state, to use in test reports + :param hooks: list of defined pre- and post-step hooks + :param exit_on_failure: whether or not to exit on installation failure + """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps @@ -115,7 +122,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): for ec in ecs: ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -455,7 +462,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=load_hooks(), + exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 618ff7bc34..6718af4fee 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1654,14 +1654,9 @@ def diff_files(path1, path2): return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2)) -def find_hook(step_name, pre_hook=True): - """ - Find pre- or post-hook for specified step. - - :param step_name: name of the step that hook relates to - :param pre_hook: True to search for pre-step hook, False to search for post-step hook - """ - res = None +def load_hooks(): + """Load defined hooks (if any).""" + hooks = [] hooks_path = build_option('hooks') if hooks_path: (hooks_dir, hooks_filename) = os.path.split(hooks_path) @@ -1670,19 +1665,38 @@ def find_hook(step_name, pre_hook=True): _log.info("Importing hooks implementation from %s...", hooks_path) (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir]) try: + # import module that defines hooks, and collect all functions of which name ends with '_hook' imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr) - found_hooks = [attr for attr in dir(imported_hooks) if attr.endswith('_hook')] - _log.debug("Found hooks: %s", found_hooks) + for attr in dir(imported_hooks): + if attr.endswith('_hook'): + hook = getattr(imported_hooks, attr) + if callable(hook): + hooks.append(hook) + else: + _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) + _log.debug("Found hooks: %s", hooks) except ImportError as err: raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) - - hook_name = ('post_', 'pre_')[pre_hook] + step_name + '_hook' - if hook_name in found_hooks: - res = getattr(imported_hooks, hook_name) - print "Found %s: %s" % (hook_name, res) else: raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)") else: _log.info("No location for hooks implementation provided, no hooks defined") + return hooks + + +def find_hook(step_name, known_hooks, pre_hook=True): + """ + Find pre- or post-hook for specified step. + + :param step_name: name of the step that hook relates to + :param pre_hook: True to search for pre-step hook, False to search for post-step hook + """ + res = None + hook_name = ('post_', 'pre_')[pre_hook] + step_name + '_hook' + for hook in known_hooks: + if hook.__name__ == hook_name: + res = hook + break + return res From 1bc7dceeebef677e358345a80a88b4e4ba1980f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Nov 2017 11:46:29 +0100 Subject: [PATCH 031/256] add tests for load_hooks and find_hook functions --- easybuild/framework/easyblock.py | 2 +- easybuild/main.py | 3 ++- easybuild/tools/filetools.py | 3 +-- test/framework/filetools.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8386bab8a1..a57fab50bd 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -145,7 +145,7 @@ def __init__(self, ec, hooks=None): self.orig_workdir = os.getcwd() # list of pre- and post-step hooks - self.hooks = hooks + self.hooks = hooks or [] # list of patch/source files, along with checksums self.patches = [] diff --git a/easybuild/main.py b/easybuild/main.py index 54814f7982..b1a59ed30d 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -460,9 +460,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): sys.exit(0) # build software, will exit when errors occurs (except when testing) + hooks = load_hooks(build_option('hooks')) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=load_hooks(), + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=hooks, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6718af4fee..8b8eeb8a75 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1654,10 +1654,9 @@ def diff_files(path1, path2): return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2)) -def load_hooks(): +def load_hooks(hooks_path): """Load defined hooks (if any).""" hooks = [] - hooks_path = build_option('hooks') if hooks_path: (hooks_dir, hooks_filename) = os.path.split(hooks_path) (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8a13afb1ae..90165bbe88 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1542,6 +1542,36 @@ def test_diff_files(self): regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res)) + def test_hooks(self): + """Test for functions that support use of hooks.""" + test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py') + test_hooks_pymod_txt = '\n'.join([ + 'def pre_install_hook(self):', + ' pass', + 'def foo():', + ' pass', + 'def post_configure_hook(self):', + ' foo()', + ' pass', + ]) + ft.write_file(test_hooks_pymod, test_hooks_pymod_txt) + + hooks = ft.load_hooks(test_hooks_pymod) + self.assertEqual(len(hooks), 2) + self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook']) + self.assertTrue(all(callable(h) for h in hooks)) + + post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] + pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] + + self.assertEqual(ft.find_hook('configure', hooks), None) + self.assertEqual(ft.find_hook('configure', hooks, pre_hook=True), None) + self.assertEqual(ft.find_hook('configure', hooks, pre_hook=False), post_configure_hook) + + self.assertEqual(ft.find_hook('install', hooks), pre_install_hook) + self.assertEqual(ft.find_hook('install', hooks, pre_hook=True), pre_install_hook) + self.assertEqual(ft.find_hook('install', hooks, pre_hook=False), None) + def suite(): """ returns all the testcases in this module """ From e5477ac164146907690031037e19c872b1a7feea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Nov 2017 13:37:45 +0100 Subject: [PATCH 032/256] add test for --hooks option --- test/framework/toy_build.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9427b247d0..7dae57a900 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1743,6 +1743,41 @@ def test_toy_build_trace(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def test_toy_build_hooks(self): + """Test use of --hooks.""" + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = '\n'.join([ + "import os", + '', + "def pre_configure_hook(self):", + " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", + '', + "def post_configure_hook(self):", + " print('post-configure: toy.source: %s' % os.path.exists('toy.source'))", + '', + "def post_install_hook(self):", + " print('in post-install hook for %s v%s' % (self.name, self.version))", + " print(', '.join(sorted(os.listdir(self.installdir))))", + ]) + write_file(hooks_file, hooks_file_txt) + + self.mock_stderr(True) + self.mock_stdout(True) + self.test_toy_build(extra_args=['--hooks=%s' % hooks_file]) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + expected_output = '\n'.join([ + "pre-configure: toy.source: True", + "post-configure: toy.source: False", + "in post-install hook for toy v0.0", + "bin, lib", + ]) + self.assertEqual(stdout.strip(), expected_output) + def suite(): """ return all the tests in this file """ From 06ba53c74da311edff911a6b1c861436bbb1249d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Nov 2017 09:01:12 +0100 Subject: [PATCH 033/256] install paramiko version < 2.4.0 for Python 2.6 in Travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a2d757fa24..64b84d8962 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,8 @@ before_install: - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'GitPython<2.0'; else pip install GitPython; fi # pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6 - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install pydot==1.1.0; else pip install pydot; fi + # paramiko (dep for GC3Pie) 2.4.0 & more recent doesn't work with Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'paramiko<2.4.0'; else pip install paramiko; fi # optional Python packages for EasyBuild - pip install autopep8 GC3Pie pycodestyle python-graph-dot python-hglib PyYAML # git config is required to make actual git commits (cfr. tests for GitRepository) From 347ec4112d30e3cbfd2affecfec969183dae8432 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Nov 2017 15:34:22 +0100 Subject: [PATCH 034/256] add support for 'exts_default_options' easyconfig parameter --- easybuild/framework/easyblock.py | 13 +++++--- easybuild/framework/easyconfig/default.py | 1 + test/framework/easyconfig.py | 31 ++++++++++++++----- .../t/toy/toy-0.0-gompi-1.3.12-test.eb | 6 +++- test/framework/options.py | 4 ++- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6101dd331a..f9966699d7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -470,14 +470,19 @@ def fetch_extension_sources(self, skip_checksums=False): exts_sources.append({'name': ext_name}) else: ext_version = ext[1] - ext_options = {} + + # make sure we grab *raw* dict of default options for extension, + # since it may use template values like %(name)s & %(version)s + self.cfg.enable_templating = False + ext_options = copy.deepcopy(self.cfg['exts_default_options']) + self.cfg.enable_templating = True def_src_tmpl = "%(name)s-%(version)s.tar.gz" if len(ext) == 3: - ext_options = ext[2] - - if not isinstance(ext_options, dict): + if isinstance(ext_options, dict): + ext_options.update(ext[2]) + else: raise EasyBuildError("Unexpected type (non-dict) for 3rd element of %s", ext) elif len(ext) > 3: raise EasyBuildError('Extension specified in unknown format (list/tuple too long)') diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 2458e830ab..864985b1d0 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -153,6 +153,7 @@ # EXTENSIONS easyconfig parameters 'exts_classmap': [{}, "Map of extension name to class for handling build and installation.", EXTENSIONS], 'exts_defaultclass': [None, "List of module for and name of the default extension class", EXTENSIONS], + 'exts_default_options': [{}, "List of default options for extensions", EXTENSIONS], 'exts_filter': [None, ("Extension filter details: template for cmd and input to cmd " "(templates for name, version and src)."), EXTENSIONS], 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 2377ba6885..c3c40518e7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -310,13 +310,13 @@ def test_exts_list(self): 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', + 'exts_default_options = {', + ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail + ' "source_urls": ["http://example.com/"]', + '}', 'exts_list = [', - ' ("ext1", "ext_ver1", {', - ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail - ' "source_urls": ["http://example.com/"]', - ' }),', - ' ("ext2", "ext_ver2", {', - ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail + ' ("ext1", "1.0"),', + ' ("ext2", "2.0", {', ' "source_urls": [("http://example.com", "suffix")],' ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail ' "checksums": [', @@ -333,9 +333,26 @@ def test_exts_list(self): eb = EasyBlock(ec) exts_sources = eb.fetch_extension_sources() + self.assertEqual(len(exts_sources), 2) + self.assertEqual(exts_sources[0]['name'], 'ext1') + self.assertEqual(exts_sources[0]['version'], '1.0') + self.assertEqual(exts_sources[0]['options'], { + 'source_tmpl': 'gzip-1.4.eb', + 'source_urls': ['http://example.com/'], + }) + self.assertEqual(exts_sources[1]['name'], 'ext2') + self.assertEqual(exts_sources[1]['version'], '2.0') + self.assertEqual(exts_sources[1]['options'], { + 'checksums': ['6f281b6d7a3965476324a23b9d80232bd4ffe3967da85e4b7c01d9d81d649a09', + '044e300a051120defb01c14c7c06e9aa4bca40c5d589828df360e2684dcc9074'], + 'patches': ['toy-0.0.eb'], + 'source_tmpl': 'gzip-1.4.eb', + 'source_urls': [('http://example.com', 'suffix')], + }) + modfile = os.path.join(eb.make_module_step(), 'pi', '3.14' + eb.module_generator.MODULE_FILE_EXTENSION) modtxt = read_file(modfile) - regex = re.compile('EBEXTSLISTPI.*ext1-ext_ver1,ext2-ext_ver2') + regex = re.compile('EBEXTSLISTPI.*ext1-1.0,ext2-2.0') self.assertTrue(regex.search(modtxt), "Pattern '%s' found in: %s" % (regex.pattern, modtxt)) def test_suggestions(self): diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index 326519c88c..25c06bd4c1 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -20,12 +20,16 @@ checksums = [[ ]] patches = ['toy-0.0_typo.patch'] +exts_default_options = { + 'source_urls': ['http://example.com'], +} + exts_list = [ ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'], # checksum for # custom extension filter to verify use of stdin value being passed to filter command - 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), + 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), 'patches': ['bar-0.0_typo.patch'], 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', diff --git a/test/framework/options.py b/test/framework/options.py index 0c9369b1e8..79206ded85 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3406,11 +3406,13 @@ def test_inject_checksums(self): ], 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), 'patches': ['bar-0.0_typo.patch'], + 'source_urls': ['http://example.com'], 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', })) self.assertEqual(ec['exts_list'][1], ('barbar', '0.0', { - 'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c'] + 'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c'], + 'source_urls': ['http://example.com'], })) # backup of easyconfig was created From 909e56a2a3c4f437534ac2a8992d4473195bb779 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2017 10:20:43 +0100 Subject: [PATCH 035/256] enhance test according to @wpoely86's suggestion --- test/framework/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c3c40518e7..b07c743ebe 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -312,7 +312,7 @@ def test_exts_list(self): 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_default_options = {', ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail - ' "source_urls": ["http://example.com/"]', + ' "source_urls": ["http://example.com/%(name)s/%(version)s"]', '}', 'exts_list = [', ' ("ext1", "1.0"),', @@ -338,7 +338,7 @@ def test_exts_list(self): self.assertEqual(exts_sources[0]['version'], '1.0') self.assertEqual(exts_sources[0]['options'], { 'source_tmpl': 'gzip-1.4.eb', - 'source_urls': ['http://example.com/'], + 'source_urls': ['http://example.com/%(name)s/%(version)s'], }) self.assertEqual(exts_sources[1]['name'], 'ext2') self.assertEqual(exts_sources[1]['version'], '2.0') From 96d476c11147f553df75f79c42a4bc34e75b9d3f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2017 18:34:36 +0100 Subject: [PATCH 036/256] take exts_default_options easyconfig parameter into account for --inject-checksums --- easybuild/framework/easyblock.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 482271148f..201c66839c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3018,8 +3018,14 @@ def inject_checksums(ecs, checksum_type): if ext_options or ext_checksums: exts_list_lines[-1] += ' {' + # make sure we grab *raw* dict of default options for extension, + # since it may use template values like %(name)s & %(version)s + app.cfg.enable_templating = False + exts_default_options = app.cfg['exts_default_options'] + app.cfg.enable_templating = True + for key, val in sorted(ext_options.items()): - if key != 'checksums': + if key != 'checksums' and val != exts_default_options.get(key): val = quote_str(val, prefer_single_quotes=True) exts_list_lines.append("%s'%s': %s," % (INDENT_4SPACES * 2, key, val)) From 349cd022e072c289a66551dcbf34308e8bd5d4da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2017 18:42:26 +0100 Subject: [PATCH 037/256] enhance test for --inject-checksums to check whether exts_default_options is taken into account --- .../easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb | 2 +- test/framework/options.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index 25c06bd4c1..ef9585dde6 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -21,7 +21,7 @@ checksums = [[ patches = ['toy-0.0_typo.patch'] exts_default_options = { - 'source_urls': ['http://example.com'], + 'source_urls': ['http://example.com/%(name)s'], } exts_list = [ diff --git a/test/framework/options.py b/test/framework/options.py index 79206ded85..bd4b64a1a1 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3393,11 +3393,15 @@ def test_inject_checksums(self): # exactly three checksum specs for extensions, one list of checksums for each extension self.assertEqual(re.findall("[ ]*'checksums'", ec_txt, re.M), [" 'checksums'"] * 3) + # there should be only one hit for 'source_urls', i.e. the one in exts_default_options + self.assertEqual(len(re.findall('source_urls*.*$', ec_txt, re.M)), 1) + # no parse errors for updated easyconfig file... ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['sources'], ['%(name)s-%(version)s.tar.gz']) self.assertEqual(ec['patches'], ['toy-0.0_typo.patch']) self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) + self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']}) self.assertEqual(ec['exts_list'][0], ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': [ @@ -3406,13 +3410,11 @@ def test_inject_checksums(self): ], 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), 'patches': ['bar-0.0_typo.patch'], - 'source_urls': ['http://example.com'], 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', })) self.assertEqual(ec['exts_list'][1], ('barbar', '0.0', { 'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c'], - 'source_urls': ['http://example.com'], })) # backup of easyconfig was created From ff9bd887645bdaf1018206455a69e1ba3bab8cdf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2017 15:42:44 +0100 Subject: [PATCH 038/256] rename 'find_hook' to 'find_step_hook' and add more generic 'find_hook' function --- easybuild/framework/easyblock.py | 6 +++--- easybuild/tools/filetools.py | 23 +++++++++++++++++------ test/framework/filetools.py | 16 ++++++++++------ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a57fab50bd..a4fa6a8609 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, find_hook, is_alt_pypi_url, mkdir, move_logs +from easybuild.tools.filetools import encode_class_name, extract_file, find_step_hook, is_alt_pypi_url, mkdir, move_logs from easybuild.tools.filetools import read_file, remove_file, rmtree2, write_file, verify_checksum, weld_paths from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml @@ -2437,7 +2437,7 @@ def run_step(self, step, step_methods): self.log.info("Starting %s step", step) self.update_config_template_run_step() - pre_hook = find_hook(step, self.hooks, pre_hook=True) + pre_hook = find_step_hook(step, self.hooks, pre_hook=True) if pre_hook: self.log.info("Found pre-%s hook, so running it...", step) pre_hook(self) @@ -2465,7 +2465,7 @@ def run_step(self, step, step_methods): # and returns the actual method, so use () to execute it step_method(self)() - post_hook = find_hook(step, self.hooks, pre_hook=False) + post_hook = find_step_hook(step, self.hooks, pre_hook=False) if post_hook: self.log.info("Found post-%s hook, so running it...", step) post_hook(self) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 8b8eeb8a75..a71a14d735 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1684,18 +1684,29 @@ def load_hooks(hooks_path): return hooks -def find_hook(step_name, known_hooks, pre_hook=True): +def find_hook(hook_name, known_hooks): """ - Find pre- or post-hook for specified step. + Find hook with specified name. - :param step_name: name of the step that hook relates to - :param pre_hook: True to search for pre-step hook, False to search for post-step hook + :param hook_name: name of hook + :param known_hooks: list of known hooks """ res = None - hook_name = ('post_', 'pre_')[pre_hook] + step_name + '_hook' for hook in known_hooks: - if hook.__name__ == hook_name: + if hook.__name__ == hook_name + '_hook': res = hook break return res + + +def find_step_hook(step_name, known_hooks, pre_hook=True): + """ + Find pre- or post-hook for specified step. + + :param step_name: name of the step that hook relates to + :param known_hooks: list of known hooks + :param pre_hook: True to search for pre-step hook, False to search for post-step hook + """ + hook_name = ('post_', 'pre_')[pre_hook] + step_name + return find_hook(hook_name, known_hooks) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 90165bbe88..6b81d53979 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1564,13 +1564,17 @@ def test_hooks(self): post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] - self.assertEqual(ft.find_hook('configure', hooks), None) - self.assertEqual(ft.find_hook('configure', hooks, pre_hook=True), None) - self.assertEqual(ft.find_hook('configure', hooks, pre_hook=False), post_configure_hook) + self.assertEqual(ft.find_step_hook('configure', hooks), None) + self.assertEqual(ft.find_step_hook('configure', hooks, pre_hook=True), None) + self.assertEqual(ft.find_step_hook('configure', hooks, pre_hook=False), post_configure_hook) - self.assertEqual(ft.find_hook('install', hooks), pre_install_hook) - self.assertEqual(ft.find_hook('install', hooks, pre_hook=True), pre_install_hook) - self.assertEqual(ft.find_hook('install', hooks, pre_hook=False), None) + self.assertEqual(ft.find_step_hook('install', hooks), pre_install_hook) + self.assertEqual(ft.find_step_hook('install', hooks, pre_hook=True), pre_install_hook) + self.assertEqual(ft.find_step_hook('install', hooks, pre_hook=False), None) + + self.assertEqual(ft.find_step_hook('build', hooks), None) + self.assertEqual(ft.find_step_hook('build', hooks, pre_hook=True), None) + self.assertEqual(ft.find_step_hook('build', hooks, pre_hook=False), None) def suite(): From 6a7dc728ffc3f2ff9181f23315bc1f8d88351a9c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2017 16:14:08 +0100 Subject: [PATCH 039/256] implement run_hook function, collapse find_hook and find_step_hook into single find_hook function --- easybuild/tools/filetools.py | 40 ++++++++++++++++------ test/framework/filetools.py | 64 +++++++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 25 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a71a14d735..138ba41a77 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1684,29 +1684,49 @@ def load_hooks(hooks_path): return hooks -def find_hook(hook_name, known_hooks): +def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): """ - Find hook with specified name. + Find hook with specified label. - :param hook_name: name of hook + :param label: name of hook :param known_hooks: list of known hooks + :param pre_step_hook: indicates whether hook to run is a pre-step hook + :param post_step_hook: indicates whether hook to run is a post-step hook """ res = None + + if pre_step_hook: + hook_prefix = 'pre_' + elif post_step_hook: + hook_prefix = 'post_' + else: + hook_prefix = '' + + hook_name = hook_prefix + label + '_hook' + for hook in known_hooks: - if hook.__name__ == hook_name + '_hook': + if hook.__name__ == hook_name: + _log.info("Found %s hook", hook_name) res = hook break return res -def find_step_hook(step_name, known_hooks, pre_hook=True): +def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None): """ - Find pre- or post-hook for specified step. + Run hook with specified label. - :param step_name: name of the step that hook relates to + :param label: name of hook :param known_hooks: list of known hooks - :param pre_hook: True to search for pre-step hook, False to search for post-step hook + :param pre_step_hook: indicates whether hook to run is a pre-step hook + :param post_step_hook: indicates whether hook to run is a post-step hook + :param args: arguments to pass to hook function """ - hook_name = ('post_', 'pre_')[pre_hook] + step_name - return find_hook(hook_name, known_hooks) + hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + if hook: + if args is None: + args = [] + + _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) + hook(*args) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 6b81d53979..d977b914ff 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1546,35 +1546,69 @@ def test_hooks(self): """Test for functions that support use of hooks.""" test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py') test_hooks_pymod_txt = '\n'.join([ - 'def pre_install_hook(self):', - ' pass', + 'def start_hook():', + ' print("running start hook")', + '', 'def foo():', - ' pass', + ' print("running foo helper method")', + '', 'def post_configure_hook(self):', + ' print("running post-configure hook")', ' foo()', - ' pass', + '', + 'def pre_install_hook(self):', + ' print("running pre-install hook")', ]) ft.write_file(test_hooks_pymod, test_hooks_pymod_txt) hooks = ft.load_hooks(test_hooks_pymod) - self.assertEqual(len(hooks), 2) - self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook']) + self.assertEqual(len(hooks), 3) + self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook', 'start_hook']) self.assertTrue(all(callable(h) for h in hooks)) post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] + start_hook = [h for h in hooks if h.__name__ == 'start_hook'][0] - self.assertEqual(ft.find_step_hook('configure', hooks), None) - self.assertEqual(ft.find_step_hook('configure', hooks, pre_hook=True), None) - self.assertEqual(ft.find_step_hook('configure', hooks, pre_hook=False), post_configure_hook) + self.assertEqual(ft.find_hook('configure', hooks), None) + self.assertEqual(ft.find_hook('configure', hooks, pre_step_hook=True), None) + self.assertEqual(ft.find_hook('configure', hooks, post_step_hook=True), post_configure_hook) - self.assertEqual(ft.find_step_hook('install', hooks), pre_install_hook) - self.assertEqual(ft.find_step_hook('install', hooks, pre_hook=True), pre_install_hook) - self.assertEqual(ft.find_step_hook('install', hooks, pre_hook=False), None) + self.assertEqual(ft.find_hook('install', hooks), None) + self.assertEqual(ft.find_hook('install', hooks, pre_step_hook=True), pre_install_hook) + self.assertEqual(ft.find_hook('install', hooks, post_step_hook=True), None) - self.assertEqual(ft.find_step_hook('build', hooks), None) - self.assertEqual(ft.find_step_hook('build', hooks, pre_hook=True), None) - self.assertEqual(ft.find_step_hook('build', hooks, pre_hook=False), None) + self.assertEqual(ft.find_hook('build', hooks), None) + self.assertEqual(ft.find_hook('build', hooks, pre_step_hook=True), None) + self.assertEqual(ft.find_hook('build', hooks, post_step_hook=True), None) + + self.assertEqual(ft.find_hook('start', hooks), start_hook) + self.assertEqual(ft.find_hook('start', hooks, pre_step_hook=True), None) + self.assertEqual(ft.find_hook('start', hooks, post_step_hook=True), None) + + self.mock_stdout(True) + self.mock_stderr(True) + ft.run_hook('start', hooks) + ft.run_hook('configure', hooks, pre_step_hook=True, args=[None]) + ft.run_hook('configure', hooks, post_step_hook=True, args=[None]) + ft.run_hook('build', hooks, pre_step_hook=True, args=[None]) + ft.run_hook('build', hooks, post_step_hook=True, args=[None]) + ft.run_hook('install', hooks, pre_step_hook=True, args=[None]) + ft.run_hook('install', hooks, post_step_hook=True, args=[None]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + expected_stdout = '\n'.join([ + "running start hook", + "running post-configure hook", + "running foo helper method", + "running pre-install hook", + ]) + + self.assertEqual(stdout.strip(), expected_stdout) + self.assertEqual(stderr, '') def suite(): From e4496e31e125f6de6c72380851fa7c7117536c9f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2017 16:14:49 +0100 Subject: [PATCH 040/256] use run_hook to run pre- and post-step hooks --- easybuild/framework/easyblock.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a4fa6a8609..3535b23fff 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,8 +70,8 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, find_step_hook, is_alt_pypi_url, mkdir, move_logs -from easybuild.tools.filetools import read_file, remove_file, rmtree2, write_file, verify_checksum, weld_paths +from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file +from easybuild.tools.filetools import remove_file, rmtree2, run_hook, write_file, verify_checksum, weld_paths from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -2437,10 +2437,7 @@ def run_step(self, step, step_methods): self.log.info("Starting %s step", step) self.update_config_template_run_step() - pre_hook = find_step_hook(step, self.hooks, pre_hook=True) - if pre_hook: - self.log.info("Found pre-%s hook, so running it...", step) - pre_hook(self) + run_hook(step, self.hooks, pre_step_hook=True, args=[self]) for step_method in step_methods: self.log.info("Running method %s part of step %s" % ('_'.join(step_method.func_code.co_names), step)) @@ -2465,10 +2462,7 @@ def run_step(self, step, step_methods): # and returns the actual method, so use () to execute it step_method(self)() - post_hook = find_step_hook(step, self.hooks, pre_hook=False) - if post_hook: - self.log.info("Found post-%s hook, so running it...", step) - post_hook(self) + run_hook(step, self.hooks, post_step_hook=True, args=[self]) if self.cfg['stop'] == step: self.log.info("Stopping after %s step.", step) From 88a50e2b7131b75b0904db7a4b84459d8e81a6b7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2017 16:15:29 +0100 Subject: [PATCH 041/256] also implement start and end hooks --- easybuild/main.py | 11 ++++++++--- test/framework/toy_build.py | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index b1a59ed30d..004cbeaf8b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -58,7 +58,7 @@ from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, load_hooks, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, load_hooks, run_hook, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.modules import modules_tool @@ -118,6 +118,8 @@ def build_and_install_software(ecs, init_session_state, hooks=None, exit_on_fail # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) + run_hook('start', hooks) + res = [] for ec in ecs: ec_res = {} @@ -161,6 +163,8 @@ def build_and_install_software(ecs, init_session_state, hooks=None, exit_on_fail res.append((ec, ec_res)) + run_hook('end', hooks) + return res @@ -460,9 +464,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): sys.exit(0) # build software, will exit when errors occurs (except when testing) - hooks = load_hooks(build_option('hooks')) - exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): + exit_on_failure = not options.dump_test_report and not options.upload_test_report + hooks = load_hooks(build_option('hooks')) + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=hooks, exit_on_failure=exit_on_failure) else: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 7dae57a900..e75c627e26 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1749,6 +1749,9 @@ def test_toy_build_hooks(self): hooks_file_txt = '\n'.join([ "import os", '', + "def start_hook():", + " print('start hook triggered')", + '', "def pre_configure_hook(self):", " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", '', @@ -1758,6 +1761,9 @@ def test_toy_build_hooks(self): "def post_install_hook(self):", " print('in post-install hook for %s v%s' % (self.name, self.version))", " print(', '.join(sorted(os.listdir(self.installdir))))", + '', + "def end_hook():", + " print('end hook triggered, all done!')", ]) write_file(hooks_file, hooks_file_txt) @@ -1771,10 +1777,12 @@ def test_toy_build_hooks(self): self.assertEqual(stderr, '') expected_output = '\n'.join([ + "start hook triggered", "pre-configure: toy.source: True", "post-configure: toy.source: False", "in post-install hook for toy v0.0", "bin, lib", + "end hook triggered, all done!", ]) self.assertEqual(stdout.strip(), expected_output) From 437d8843aea577c01ce51c2f0a71e527508a6ce1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2017 09:26:36 +0100 Subject: [PATCH 042/256] disable log rotation since it's broken anyway --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/build_log.py | 4 ++-- easybuild/tools/options.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 201c66839c..e9832e5ee3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -266,7 +266,7 @@ def _init_log(self): return self.logfile = get_log_filename(self.name, self.version, add_salt=True) - fancylogger.logToFile(self.logfile) + fancylogger.logToFile(self.logfile, max_bytes=0) self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 8bf0a6d62d..dc0f45360d 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -180,7 +180,7 @@ def exception(self, msg, *args): del _init_fancylog.manager.loggerDict[_init_fancylog.name] # we need to make sure there is a handler -fancylogger.logToFile(filename=os.devnull) +fancylogger.logToFile(filename=os.devnull, max_bytes=0) # EasyBuildLog _init_easybuildlog = fancylogger.getLogger(fname=False) @@ -196,7 +196,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) - fancylogger.logToFile(logfile) + fancylogger.logToFile(logfile, max_bytes=0) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=silent) log = fancylogger.getLogger(fname=False) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d4385ccce5..cdeaad925b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -711,7 +711,7 @@ def postprocess(self): # log to specified value of --unittest-file if self.options.unittest_file: - fancylogger.logToFile(self.options.unittest_file) + fancylogger.logToFile(self.options.unittest_file, max_bytes=0) # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) From 2a65fc415d44adae2f6a86594477fff0b3fdbb5d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2017 10:46:16 +0100 Subject: [PATCH 043/256] avoid duplicate logging of executed commands under --debug --- easybuild/tools/run.py | 10 +++++----- test/framework/run.py | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 67689bb64f..d0a94e6ffc 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -475,17 +475,17 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): if not regexp: use_regexp = False - _log.debug('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) - if ec and (log_all or log_ok): # We don't want to error if the user doesn't care if check_ec: - raise EasyBuildError('cmd "%s" exited with exitcode %s and output:\n%s', cmd, ec, stdouterr) + raise EasyBuildError('cmd "%s" exited with exit code %s and output:\n%s', cmd, ec, stdouterr) else: - _log.warn('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + _log.warn('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) elif not ec: if log_all: - _log.info('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + _log.info('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) + else: + _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr)) # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in if use_regexp or regexp: diff --git a/test/framework/run.py b/test/framework/run.py index b548c6131b..e05f18d2a1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -35,15 +35,15 @@ import signal import stat import sys +import tempfile from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from vsc.utils.fancylogger import setLogLevelDebug, logToScreen import easybuild.tools.utilities -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.filetools import adjust_permissions, read_file, write_file from easybuild.tools.run import run_cmd, run_cmd_qa, parse_log_for_error -from easybuild.tools.run import _log as run_log class RunTest(EnhancedTestCase): @@ -68,6 +68,41 @@ def test_run_cmd(self): # no reason echo hello could fail self.assertEqual(ec, 0) + def test_run_cmd_log(self): + """Test logging of executed commands.""" + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') + os.close(fd) + + regex = re.compile('cmd "echo hello" exited with exit code [0-9]* and output:') + + # command output is not logged by default without debug logging + init_logging(logfile, silent=True) + self.assertTrue(run_cmd("echo hello")) + stop_logging(logfile) + self.assertEqual(len(regex.findall(read_file(logfile))), 0) + write_file(logfile, '') + + init_logging(logfile, silent=True) + self.assertTrue(run_cmd("echo hello", log_all=True)) + stop_logging(logfile) + self.assertEqual(len(regex.findall(read_file(logfile))), 1) + write_file(logfile, '') + + # with debugging enabled, exit code and output of command should only get logged once + setLogLevelDebug() + + init_logging(logfile, silent=True) + self.assertTrue(run_cmd("echo hello")) + stop_logging(logfile) + self.assertEqual(len(regex.findall(read_file(logfile))), 1) + write_file(logfile, '') + + init_logging(logfile, silent=True) + self.assertTrue(run_cmd("echo hello", log_all=True)) + stop_logging(logfile) + self.assertEqual(len(regex.findall(read_file(logfile))), 1) + write_file(logfile, '') + def test_run_cmd_negative_exit_code(self): """Test run_cmd function with command that has negative exit code.""" # define signal handler to call in case run_cmd takes too long From 32e69e557c07d3e5ca2d625833a291e49a14bbab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2017 11:17:43 +0100 Subject: [PATCH 044/256] skip testing of --preview-pr if not GitHub token is available --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index bd4b64a1a1..edf15fe62e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2187,6 +2187,9 @@ def test_cleanup_tmpdir(self): def test_preview_pr(self): """Test --preview-pr.""" + if self.github_token is None: + print "Skipping test_preview_pr, no GitHub token available?" + return self.mock_stdout(True) From 779ebfea56c3c1b7c98a1ae7f1f304198ed7bbc9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Nov 2017 20:52:44 +0100 Subject: [PATCH 045/256] fix order in import statement --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3535b23fff..c8a615d044 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -71,7 +71,7 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file -from easybuild.tools.filetools import remove_file, rmtree2, run_hook, write_file, verify_checksum, weld_paths +from easybuild.tools.filetools import remove_file, rmtree2, run_hook, verify_checksum, weld_paths, write_file from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for From a9f55d48bc0421e2134f45b778f76603bbc52f27 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 Nov 2017 08:29:27 +0100 Subject: [PATCH 046/256] flesh out hook support into easybuild.tools.hooks module --- easybuild/framework/easyblock.py | 3 +- easybuild/main.py | 3 +- easybuild/tools/filetools.py | 80 +------------------ easybuild/tools/hooks.py | 113 ++++++++++++++++++++++++++ test/framework/filetools.py | 68 ---------------- test/framework/hooks.py | 131 +++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 149 deletions(-) create mode 100644 easybuild/tools/hooks.py create mode 100644 test/framework/hooks.py diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c8a615d044..8221198671 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -71,7 +71,8 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file -from easybuild.tools.filetools import remove_file, rmtree2, run_hook, verify_checksum, weld_paths, write_file +from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file +from easybuild.tools.hooks import run_hook from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for diff --git a/easybuild/main.py b/easybuild/main.py index 004cbeaf8b..8840d7f1d1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -58,9 +58,10 @@ from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, load_hooks, run_hook, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr +from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 138ba41a77..19c8bd55c4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,4 +1,4 @@ - +# # # Copyright 2009-2017 Ghent University # # This file is part of EasyBuild, @@ -1652,81 +1652,3 @@ def diff_files(path1, path2): file1_lines = ['%s\n' % l for l in read_file(path1).split('\n')] file2_lines = ['%s\n' % l for l in read_file(path2).split('\n')] return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2)) - - -def load_hooks(hooks_path): - """Load defined hooks (if any).""" - hooks = [] - if hooks_path: - (hooks_dir, hooks_filename) = os.path.split(hooks_path) - (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) - if hooks_file_ext == '.py': - _log.info("Importing hooks implementation from %s...", hooks_path) - (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir]) - try: - # import module that defines hooks, and collect all functions of which name ends with '_hook' - imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr) - for attr in dir(imported_hooks): - if attr.endswith('_hook'): - hook = getattr(imported_hooks, attr) - if callable(hook): - hooks.append(hook) - else: - _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) - _log.debug("Found hooks: %s", hooks) - except ImportError as err: - raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) - else: - raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)") - else: - _log.info("No location for hooks implementation provided, no hooks defined") - - return hooks - - -def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): - """ - Find hook with specified label. - - :param label: name of hook - :param known_hooks: list of known hooks - :param pre_step_hook: indicates whether hook to run is a pre-step hook - :param post_step_hook: indicates whether hook to run is a post-step hook - """ - res = None - - if pre_step_hook: - hook_prefix = 'pre_' - elif post_step_hook: - hook_prefix = 'post_' - else: - hook_prefix = '' - - hook_name = hook_prefix + label + '_hook' - - for hook in known_hooks: - if hook.__name__ == hook_name: - _log.info("Found %s hook", hook_name) - res = hook - break - - return res - - -def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None): - """ - Run hook with specified label. - - :param label: name of hook - :param known_hooks: list of known hooks - :param pre_step_hook: indicates whether hook to run is a pre-step hook - :param post_step_hook: indicates whether hook to run is a post-step hook - :param args: arguments to pass to hook function - """ - hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) - if hook: - if args is None: - args = [] - - _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) - hook(*args) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py new file mode 100644 index 0000000000..930f4f5bd2 --- /dev/null +++ b/easybuild/tools/hooks.py @@ -0,0 +1,113 @@ +# # +# Copyright 2017-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Hook support. + +:author: Kenneth Hoste (Ghent University) +""" +import imp +import os +from vsc.utils import fancylogger + + +_log = fancylogger.getLogger('hooks', fname=False) + + +def load_hooks(hooks_path): + """Load defined hooks (if any).""" + hooks = [] + if hooks_path: + (hooks_dir, hooks_filename) = os.path.split(hooks_path) + (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) + if hooks_file_ext == '.py': + _log.info("Importing hooks implementation from %s...", hooks_path) + (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir]) + try: + # import module that defines hooks, and collect all functions of which name ends with '_hook' + imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr) + for attr in dir(imported_hooks): + if attr.endswith('_hook'): + hook = getattr(imported_hooks, attr) + if callable(hook): + hooks.append(hook) + else: + _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) + _log.debug("Found hooks: %s", hooks) + except ImportError as err: + raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) + else: + raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)") + else: + _log.info("No location for hooks implementation provided, no hooks defined") + + return hooks + + +def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): + """ + Find hook with specified label. + + :param label: name of hook + :param known_hooks: list of known hooks + :param pre_step_hook: indicates whether hook to run is a pre-step hook + :param post_step_hook: indicates whether hook to run is a post-step hook + """ + res = None + + if pre_step_hook: + hook_prefix = 'pre_' + elif post_step_hook: + hook_prefix = 'post_' + else: + hook_prefix = '' + + hook_name = hook_prefix + label + '_hook' + + for hook in known_hooks: + if hook.__name__ == hook_name: + _log.info("Found %s hook", hook_name) + res = hook + break + + return res + + +def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None): + """ + Run hook with specified label. + + :param label: name of hook + :param known_hooks: list of known hooks + :param pre_step_hook: indicates whether hook to run is a pre-step hook + :param post_step_hook: indicates whether hook to run is a post-step hook + :param args: arguments to pass to hook function + """ + hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + if hook: + if args is None: + args = [] + + _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) + hook(*args) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d977b914ff..8a13afb1ae 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1542,74 +1542,6 @@ def test_diff_files(self): regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res)) - def test_hooks(self): - """Test for functions that support use of hooks.""" - test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py') - test_hooks_pymod_txt = '\n'.join([ - 'def start_hook():', - ' print("running start hook")', - '', - 'def foo():', - ' print("running foo helper method")', - '', - 'def post_configure_hook(self):', - ' print("running post-configure hook")', - ' foo()', - '', - 'def pre_install_hook(self):', - ' print("running pre-install hook")', - ]) - ft.write_file(test_hooks_pymod, test_hooks_pymod_txt) - - hooks = ft.load_hooks(test_hooks_pymod) - self.assertEqual(len(hooks), 3) - self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook', 'start_hook']) - self.assertTrue(all(callable(h) for h in hooks)) - - post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] - pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] - start_hook = [h for h in hooks if h.__name__ == 'start_hook'][0] - - self.assertEqual(ft.find_hook('configure', hooks), None) - self.assertEqual(ft.find_hook('configure', hooks, pre_step_hook=True), None) - self.assertEqual(ft.find_hook('configure', hooks, post_step_hook=True), post_configure_hook) - - self.assertEqual(ft.find_hook('install', hooks), None) - self.assertEqual(ft.find_hook('install', hooks, pre_step_hook=True), pre_install_hook) - self.assertEqual(ft.find_hook('install', hooks, post_step_hook=True), None) - - self.assertEqual(ft.find_hook('build', hooks), None) - self.assertEqual(ft.find_hook('build', hooks, pre_step_hook=True), None) - self.assertEqual(ft.find_hook('build', hooks, post_step_hook=True), None) - - self.assertEqual(ft.find_hook('start', hooks), start_hook) - self.assertEqual(ft.find_hook('start', hooks, pre_step_hook=True), None) - self.assertEqual(ft.find_hook('start', hooks, post_step_hook=True), None) - - self.mock_stdout(True) - self.mock_stderr(True) - ft.run_hook('start', hooks) - ft.run_hook('configure', hooks, pre_step_hook=True, args=[None]) - ft.run_hook('configure', hooks, post_step_hook=True, args=[None]) - ft.run_hook('build', hooks, pre_step_hook=True, args=[None]) - ft.run_hook('build', hooks, post_step_hook=True, args=[None]) - ft.run_hook('install', hooks, pre_step_hook=True, args=[None]) - ft.run_hook('install', hooks, post_step_hook=True, args=[None]) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) - - expected_stdout = '\n'.join([ - "running start hook", - "running post-configure hook", - "running foo helper method", - "running pre-install hook", - ]) - - self.assertEqual(stdout.strip(), expected_stdout) - self.assertEqual(stderr, '') - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/hooks.py b/test/framework/hooks.py new file mode 100644 index 0000000000..b1b12ddc18 --- /dev/null +++ b/test/framework/hooks.py @@ -0,0 +1,131 @@ +# # +# Copyright 2017-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for hooks.py + +@author: Kenneth Hoste (Ghent University) +""" +import os +import sys +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from unittest import TextTestRunner + +from easybuild.tools.filetools import write_file +from easybuild.tools.hooks import find_hook, load_hooks, run_hook + + +class HooksTest(EnhancedTestCase): + """Tests for hooks support.""" + + def setUp(self): + """Set up for testing.""" + super(HooksTest, self).setUp() + self.test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py') + test_hooks_pymod_txt = '\n'.join([ + 'def start_hook():', + ' print("running start hook")', + '', + 'def foo():', + ' print("running foo helper method")', + '', + 'def post_configure_hook(self):', + ' print("running post-configure hook")', + ' foo()', + '', + 'def pre_install_hook(self):', + ' print("running pre-install hook")', + ]) + write_file(self.test_hooks_pymod, test_hooks_pymod_txt) + + def testload__hooks(self): + """Test for load_hooks function.""" + + hooks = load_hooks(self.test_hooks_pymod) + + self.assertEqual(len(hooks), 3) + self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook', 'start_hook']) + self.assertTrue(all(callable(h) for h in hooks)) + + def test_find_hook(self): + """Test for find_hook function.""" + + hooks = load_hooks(self.test_hooks_pymod) + + post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] + pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] + start_hook = [h for h in hooks if h.__name__ == 'start_hook'][0] + + self.assertEqual(find_hook('configure', hooks), None) + self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('configure', hooks, post_step_hook=True), post_configure_hook) + + self.assertEqual(find_hook('install', hooks), None) + self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook) + self.assertEqual(find_hook('install', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('build', hooks), None) + self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('build', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('start', hooks), start_hook) + self.assertEqual(find_hook('start', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('start', hooks, post_step_hook=True), None) + + def test_run_hook(self): + """Test for run_hook function.""" + + hooks = load_hooks(self.test_hooks_pymod) + + self.mock_stdout(True) + self.mock_stderr(True) + run_hook('start', hooks) + run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('configure', hooks, post_step_hook=True, args=[None]) + run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('build', hooks, post_step_hook=True, args=[None]) + run_hook('install', hooks, pre_step_hook=True, args=[None]) + run_hook('install', hooks, post_step_hook=True, args=[None]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + expected_stdout = '\n'.join([ + "running start hook", + "running post-configure hook", + "running foo helper method", + "running pre-install hook", + ]) + + self.assertEqual(stdout.strip(), expected_stdout) + self.assertEqual(stderr, '') + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(HooksTest, sys.argv[1:]) + +if __name__ == '__main__': + TextTestRunner(verbosity=1).run(suite()) From 3e4b326d73cf1a3e1d33729305ebd12d6c248614 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 Nov 2017 08:39:26 +0100 Subject: [PATCH 047/256] no need to resolve --hooks path or make it a build option, but do make sure specified path exists --- easybuild/main.py | 2 +- easybuild/tools/config.py | 1 - easybuild/tools/hooks.py | 6 ++++++ easybuild/tools/options.py | 6 +----- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8840d7f1d1..85537141c6 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -467,7 +467,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not options.dump_test_report and not options.upload_test_report - hooks = load_hooks(build_option('hooks')) + hooks = load_hooks(options.hooks) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=hooks, exit_on_failure=exit_on_failure) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 05f4b82370..5f133b6604 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -121,7 +121,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'force_download', 'from_pr', 'git_working_dirs_path', - 'hooks', 'github_user', 'github_org', 'group', diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 930f4f5bd2..1acf7cf24d 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -31,6 +31,8 @@ import os from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('hooks', fname=False) @@ -38,7 +40,11 @@ def load_hooks(hooks_path): """Load defined hooks (if any).""" hooks = [] + if hooks_path: + if not os.path.exists(hooks_path): + raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path) + (hooks_dir, hooks_filename) = os.path.split(hooks_path) (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) if hooks_file_ext == '.py': diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a366c81fdc..6db475fdcd 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -70,7 +70,7 @@ from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars -from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir, resolve_path +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, HAVE_GITHUB_API, HAVE_KEYRING from easybuild.tools.github import fetch_github_token from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains @@ -780,10 +780,6 @@ def postprocess(self): # handle configuration options that affect other configuration options self._postprocess_config() - # make sure absolute path is used for location of hooks - if self.options.hooks: - self.options.hooks = resolve_path(self.options.hooks) - # show current configuration and exit, if requested if self.options.show_config or self.options.show_full_config: self.show_config() From fb820020cf3389e0f101f3cb5b31ed86fde29122 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 Nov 2017 09:27:35 +0100 Subject: [PATCH 048/256] verify defined hooks and produce error if unknown hooks are defined --- easybuild/tools/hooks.py | 40 ++++++++++++++++++++++++++++++---------- test/framework/hooks.py | 31 +++++++++++++++++++++++++++++-- test/framework/suite.py | 3 ++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 1acf7cf24d..f604aaaaef 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -36,6 +36,12 @@ _log = fancylogger.getLogger('hooks', fname=False) +# this should be obtained via EasyBlock.get_steps(), +# but we can't import from easybuild.framework.easyblock without introducing a cyclic dependency... +STEP_NAMES = ['fetch', 'ready', 'source', 'patch', 'prepare', 'configure', 'build', 'test', 'install', 'extensions', + 'postproc', 'sanitycheck', 'cleanup', 'module', 'permissions', 'package', 'testcases'] +KNOWN_HOOKS = ['%s_hook' % h for h in ['start'] + [p + '_' + s for s in STEP_NAMES for p in ['pre', 'post']] + ['end']] + def load_hooks(hooks_path): """Load defined hooks (if any).""" @@ -45,14 +51,12 @@ def load_hooks(hooks_path): if not os.path.exists(hooks_path): raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path) - (hooks_dir, hooks_filename) = os.path.split(hooks_path) - (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename) + (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1]) if hooks_file_ext == '.py': _log.info("Importing hooks implementation from %s...", hooks_path) - (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir]) try: # import module that defines hooks, and collect all functions of which name ends with '_hook' - imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr) + imported_hooks = imp.load_source(hooks_filename, hooks_path) for attr in dir(imported_hooks): if attr.endswith('_hook'): hook = getattr(imported_hooks, attr) @@ -68,15 +72,31 @@ def load_hooks(hooks_path): else: _log.info("No location for hooks implementation provided, no hooks defined") + verify_hooks(hooks) + return hooks -def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): +def verify_hooks(hooks): + """Check whether list of obtained hooks only includes known hooks.""" + unknown_hooks = [] + for hook in hooks: + if hook.__name__ not in KNOWN_HOOKS: + unknown_hooks.append(hook.__name__) + + if unknown_hooks: + raise EasyBuildError("Found one or more unknown hooks: %s (known hooks: %s)", + ', '.join(unknown_hooks), ', '.join(KNOWN_HOOKS)) + else: + _log.info("Defined hooks verified, all known hooks: %s", ', '.join(h.__name__ for h in hooks)) + + +def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): """ Find hook with specified label. :param label: name of hook - :param known_hooks: list of known hooks + :param hooks: list of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook """ @@ -91,7 +111,7 @@ def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): hook_name = hook_prefix + label + '_hook' - for hook in known_hooks: + for hook in hooks: if hook.__name__ == hook_name: _log.info("Found %s hook", hook_name) res = hook @@ -100,17 +120,17 @@ def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False): return res -def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None): +def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None): """ Run hook with specified label. :param label: name of hook - :param known_hooks: list of known hooks + :param hooks: list of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook :param args: arguments to pass to hook function """ - hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) if hook: if args is None: args = [] diff --git a/test/framework/hooks.py b/test/framework/hooks.py index b1b12ddc18..10c0d242ea 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -32,8 +32,9 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file -from easybuild.tools.hooks import find_hook, load_hooks, run_hook +from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks class HooksTest(EnhancedTestCase): @@ -59,9 +60,11 @@ def setUp(self): ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) - def testload__hooks(self): + def test_load_hooks(self): """Test for load_hooks function.""" + self.assertErrorRegex(EasyBuildError, "Specified path .* does not exist.*", load_hooks, '/no/such/hooks.py') + hooks = load_hooks(self.test_hooks_pymod) self.assertEqual(len(hooks), 3) @@ -122,6 +125,30 @@ def test_run_hook(self): self.assertEqual(stdout.strip(), expected_stdout) self.assertEqual(stderr, '') + def test_verify_hooks(self): + """Test verify_hooks function.""" + + hooks = load_hooks(self.test_hooks_pymod) + # verify_hooks is actually already called by load_hooks, so this is a bit silly, but fine + # if no unexpected hooks are found, verify_hooks just logs (no return value) + self.assertEqual(verify_hooks(hooks), None) + + test_broken_hooks_pymod = os.path.join(self.test_prefix, 'test_broken_hooks.py') + test_hooks_txt = '\n'.join([ + '', + 'def there_is_no_such_hook():', + ' pass', + 'def post_source_hook(self):', + ' pass', + 'def another_faulty_hook(self):', + ' pass', + ]) + + write_file(test_broken_hooks_pymod, test_hooks_txt) + + error_msg_pattern = "Found one or more unknown hooks: another_faulty_hook, there_is_no_such_hook \(known hooks:" + self.assertErrorRegex(EasyBuildError, error_msg_pattern, load_hooks, test_broken_hooks_pymod) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/suite.py b/test/framework/suite.py index ecd007f97e..bd41a3ae57 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -68,6 +68,7 @@ import test.framework.format_convert as f_c import test.framework.general as gen import test.framework.github as g +import test.framework.hooks as h import test.framework.include as i import test.framework.license as l import test.framework.module_generator as mg @@ -108,7 +109,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, - tw, p, i, pkg, d, env, et, y, st] + tw, p, i, pkg, d, env, et, y, st, h] SUITE = unittest.TestSuite([x.suite() for x in tests]) From e84f51b2d1ecb41ebd50b25f26fe53d9e2c834b5 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Mon, 27 Nov 2017 11:40:02 +0100 Subject: [PATCH 049/256] Allow use of `start_dir` in extensions --- easybuild/framework/extensioneasyblock.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 789c44d3f0..8b4b54d339 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -32,7 +32,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extension import Extension from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import apply_patch, extract_file +from easybuild.tools.filetools import apply_patch, change_dir, extract_file from easybuild.tools.utilities import remove_unwanted_chars @@ -101,6 +101,10 @@ def run(self, unpack_src=False): targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file("%s" % self.src, targetdir, extra_options=self.unpack_options) + if self.cfg['start_dir'] and os.path.isdir(self.cfg['start_dir']): + self.log.debug("Using start_dir: %s", self.cfg['start_dir']) + change_dir(self.cfg['start_dir']) + # patch if needed if self.patches: for patchfile in self.patches: From 4cf703dc25847f7be9ff582e212852dda70fe04c Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Mon, 27 Nov 2017 14:34:45 +0100 Subject: [PATCH 050/256] Fix bug: don't blind use the `start_dir` from the parent --- easybuild/framework/extensioneasyblock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 8b4b54d339..8b098cf879 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -81,6 +81,7 @@ def __init__(self, *args, **kwargs): # name and version properties of EasyBlock are used, so make sure name and version are correct self.cfg['name'] = self.ext.get('name', None) self.cfg['version'] = self.ext.get('version', None) + self.cfg['start_dir'] = self.ext.get('options', {}).get('start_dir', None) self.builddir = self.master.builddir self.installdir = self.master.installdir self.modules_tool = self.master.modules_tool From 2853ec854f189d24d8808fad866332ec7ac6c11d Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 28 Nov 2017 10:15:46 +0100 Subject: [PATCH 051/256] Use .start_dir --- easybuild/framework/extensioneasyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 8b098cf879..97c7dbe3cd 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -102,9 +102,9 @@ def run(self, unpack_src=False): targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file("%s" % self.src, targetdir, extra_options=self.unpack_options) - if self.cfg['start_dir'] and os.path.isdir(self.cfg['start_dir']): - self.log.debug("Using start_dir: %s", self.cfg['start_dir']) - change_dir(self.cfg['start_dir']) + if self.start_dir and os.path.isdir(self.start_dir]): + self.log.debug("Using start_dir: %s", self.start_dir) + change_dir(self.start_dir) # patch if needed if self.patches: From e9a4830737045d946187377c34279a4e7db43951 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 28 Nov 2017 10:18:56 +0100 Subject: [PATCH 052/256] Add comment to clarify the situation --- easybuild/framework/extensioneasyblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 97c7dbe3cd..7d7575099a 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -81,6 +81,8 @@ def __init__(self, *args, **kwargs): # name and version properties of EasyBlock are used, so make sure name and version are correct self.cfg['name'] = self.ext.get('name', None) self.cfg['version'] = self.ext.get('version', None) + # We cannot inherit the start_dir from the easyconfig. It should be specified in the extension + # itself or be empty. self.cfg['start_dir'] = self.ext.get('options', {}).get('start_dir', None) self.builddir = self.master.builddir self.installdir = self.master.installdir From bfc44b5084c4366824b731bc30aeb5fb2514b8d4 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 28 Nov 2017 10:36:20 +0100 Subject: [PATCH 053/256] Fix typo --- easybuild/framework/extensioneasyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 7d7575099a..dc0d2c4bc7 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -104,7 +104,7 @@ def run(self, unpack_src=False): targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file("%s" % self.src, targetdir, extra_options=self.unpack_options) - if self.start_dir and os.path.isdir(self.start_dir]): + if self.start_dir and os.path.isdir(self.start_dir): self.log.debug("Using start_dir: %s", self.start_dir) change_dir(self.start_dir) From 3b1ac8c49501061138ce8cab424dcbab48eea1c2 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Wed, 29 Nov 2017 15:20:10 +0100 Subject: [PATCH 054/256] Make @boegel happy --- easybuild/framework/extensioneasyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index dc0d2c4bc7..67e497a100 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -81,8 +81,8 @@ def __init__(self, *args, **kwargs): # name and version properties of EasyBlock are used, so make sure name and version are correct self.cfg['name'] = self.ext.get('name', None) self.cfg['version'] = self.ext.get('version', None) - # We cannot inherit the start_dir from the easyconfig. It should be specified in the extension - # itself or be empty. + # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong). + # It should be specified for the extension specifically, or be empty (so it is auto-derived). self.cfg['start_dir'] = self.ext.get('options', {}).get('start_dir', None) self.builddir = self.master.builddir self.installdir = self.master.installdir From c483704c728ca00a8d9bd9f69803001391e18548 Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 30 Nov 2017 15:32:59 +0100 Subject: [PATCH 055/256] Change to RPATH injection. Add %installdir/lib, %installdir/lib64 and $ORIGIN to RPATH. This works much better with shared libraries in non-standard locations, esp. python packages --- easybuild/framework/easyblock.py | 12 +++- easybuild/scripts/rpath_args.py | 17 ++--- .../scripts/rpath_wrapper_template.sh.in | 4 +- easybuild/tools/toolchain/toolchain.py | 14 ++++- test/framework/toolchain.py | 62 +++++++++++-------- 5 files changed, 71 insertions(+), 38 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e9832e5ee3..0ed4c83910 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -195,6 +195,9 @@ def __init__(self, ec): # list of locations to include in RPATH filter used by toolchain self.rpath_filter_dirs = [] + # list of locations to include in RPATH used by toolchain + self.rpath_include_dirs = [] + # logging self.log = None self.logfile = None @@ -1734,8 +1737,15 @@ def prepare_step(self, start_dir=True): if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) + # always include self.installdir+'/lib' and self.installdir+'/lib64' and $ORIGIN + # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object + # see also https://linux.die.net/man/8/ld-linux; + self.rpath_include_dirs.append(self.installdir+'/lib') + self.rpath_include_dirs.append(self.installdir+'/lib64') + self.rpath_include_dirs.append('$ORIGIN') + # prepare toolchain: load toolchain module and dependencies, set up build environment - self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs) + self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) # handle allowed system dependencies for (name, version) in self.cfg['allow_system_deps']: diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 21df993037..8aab3d5d70 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -38,7 +38,8 @@ cmd = sys.argv[1] rpath_filter = sys.argv[2] -args = sys.argv[3:] +rpath_include = sys.argv[3] +args = sys.argv[4:] # wheter or not to use -Wl to pass options to the linker if cmd in ['ld', 'ld.gold', 'ld.bfd']: @@ -52,6 +53,8 @@ else: rpath_filter = None +rpath_include = rpath_include.split(',') + version_mode = False cmd_args, cmd_args_rpath = [], [] @@ -106,13 +109,13 @@ # add -rpath flags in front cmd_args = cmd_args_rpath + cmd_args +cmd_args_rpath = [] +for path in rpath_include: + cmd_args_rpath.append(flag_prefix + '-rpath=%s' % path) + + if not version_mode: - cmd_args = [ - # always include '$ORIGIN/../lib' and '$ORIGIN/../lib64' - # $ORIGIN will be resolved by the loader to be the full path to the 'executable' - # see also https://linux.die.net/man/8/ld-linux; - flag_prefix + '-rpath=$ORIGIN/../lib', - flag_prefix + '-rpath=$ORIGIN/../lib64', + cmd_args = cmd_args_rpath + [ # try to make sure that RUNPATH is not used by always injecting --disable-new-dtags flag_prefix + '--disable-new-dtags', ] + cmd_args diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 1f5b128e79..0d0bd08903 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -46,8 +46,8 @@ CMD=`basename $0` log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'" # rpath_args.py script spits out statement that defines $CMD_ARGS -log "%(python)s -O %(rpath_args_py)s $CMD '%(rpath_filter)s' $(echo \"$@\")'" -rpath_args_out=$(%(python)s -O %(rpath_args_py)s $CMD '%(rpath_filter)s' "$@") +log "%(python)s -O %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' $(echo \"$@\")'" +rpath_args_out=$(%(python)s -O %(rpath_args_py)s $CMD '%(rpath_filter)s' '%(rpath_include)s' "$@") log "rpath_args_out: $rpath_args_out" diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 389e7d87ba..93f7aa43bb 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -656,7 +656,7 @@ def compilers(self): return (c_comps, fortran_comps) - def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None): + def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies @@ -668,6 +668,7 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No :param silent: keep quiet, or not (mostly relates to extended dry run output) :param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) + :param rpath_include_dirs: extra directories to include in RPATH """ if loadmod: self._load_modules(silent=silent) @@ -702,7 +703,7 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No if build_option('rpath'): if self.options.get('rpath', True): - self.prepare_rpath_wrappers() + self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs) self.use_rpath = True else: self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option") @@ -764,7 +765,7 @@ def is_rpath_wrapper(path): calls_rpath_args = 'rpath_args.py $CMD' in read_file(path) return in_rpath_wrappers_dir and calls_rpath_args - def prepare_rpath_wrappers(self, rpath_filter_dirs=None): + def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands @@ -796,6 +797,12 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None): rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) + # figure out list of patterns to use in rpath include + # rpath_include = build_option('rpath_include') + rpath_include = ','.join(['%s' % d for d in rpath_include_dirs or []]) + self.log.debug("Combined RPATH includes: '%s'", rpath_include) + + # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): orig_cmd = which(cmd) @@ -824,6 +831,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None): 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, + 'rpath_include': rpath_include, 'rpath_filter': rpath_filter, 'rpath_wrapper_log': rpath_wrapper_log, } diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 76a990abaa..0bf74d1d20 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1069,8 +1069,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s gcc '' -c foo.c" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'-c'", "'foo.c'", @@ -1082,12 +1083,13 @@ def test_rpath_args_script(self): self.assertEqual(ec, 0) expected = '\n'.join([ "CMD_ARGS=('foo.o')", - "RPATH_ARGS='--disable-new-dtags -rpath=$ORIGIN/../lib -rpath=$ORIGIN/../lib64'", + "RPATH_ARGS='--disable-new-dtags -rpath=%s/lib -rpath=%s/lib64 -rpath=$ORIGIN'" % (self.test_prefix, self.test_prefix) '' ]) cmd_args = [ - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", "'--disable-new-dtags'", "'--disable-new-dtags'", "'foo.o'", @@ -1098,8 +1100,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s gcc ''" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", ] self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) @@ -1108,8 +1111,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s ld.gold '' ''" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", "'--disable-new-dtags'", "''", ] @@ -1119,8 +1123,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s gcc '' foo.c -L/foo -lfoo" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/foo'", "'foo.c'", @@ -1133,8 +1138,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s gcc '' foo.c -L../lib -lfoo" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'foo.c'", "'-L../lib'", @@ -1146,8 +1152,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s gcc '' foo.c -L /foo -lfoo" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/foo'", "'foo.c'", @@ -1160,8 +1167,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s ld '' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", "'--disable-new-dtags'", "'-rpath=/foo'", "'-rpath=/lib64'", @@ -1181,8 +1189,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s ld '/fo.*,/bar.*' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % script, simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-rpath=$ORIGIN/../lib'", - "'-rpath=$ORIGIN/../lib64'", + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", "'--disable-new-dtags'", "'-rpath=/lib64'", "'-L/foo'", @@ -1214,8 +1223,9 @@ def test_rpath_args_script(self): out, ec = run_cmd("%s icc '' %s" % (script, args), simple=False) self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/icc/lib/intel64'", "'-Wl,-rpath=/imkl/lib'", @@ -1256,8 +1266,9 @@ def test_rpath_args_script(self): self.assertEqual(ec, 0) cmd_args = [ - "'-Wl,-rpath=$ORIGIN/../lib'", - "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", "'-Wl,--disable-new-dtags'", "'-DHAVE_CONFIG_H'", "'-I.'", @@ -1325,8 +1336,9 @@ def test_toolchain_prepare_rpath(self): # no -rpath for /bar because of rpath filter out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""') expected = ' '.join([ - '-Wl,-rpath=$ORIGIN/../lib', - '-Wl,-rpath=$ORIGIN/../lib64', + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", '-Wl,--disable-new-dtags', '-Wl,-rpath=/foo', '%(user)s.c', From 0a74817f496e5b47a28bd45f932a490558dec637 Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 30 Nov 2017 16:55:14 +0100 Subject: [PATCH 056/256] Added comma --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0bf74d1d20..0f5d842f70 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1083,7 +1083,7 @@ def test_rpath_args_script(self): self.assertEqual(ec, 0) expected = '\n'.join([ "CMD_ARGS=('foo.o')", - "RPATH_ARGS='--disable-new-dtags -rpath=%s/lib -rpath=%s/lib64 -rpath=$ORIGIN'" % (self.test_prefix, self.test_prefix) + "RPATH_ARGS='--disable-new-dtags -rpath=%s/lib -rpath=%s/lib64 -rpath=$ORIGIN'" % (self.test_prefix, self.test_prefix), '' ]) cmd_args = [ From d510fbb49ca51897d304a05b03e60d72eb60dd0e Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 30 Nov 2017 17:46:34 +0100 Subject: [PATCH 057/256] Adjusted tests to new rpath_args.py --- test/framework/toolchain.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0f5d842f70..5189338a3a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1066,7 +1066,7 @@ def test_rpath_args_script(self): script = find_eb_script('rpath_args.py') # simplest possible compiler command - out, ec = run_cmd("%s gcc '' -c foo.c" % script, simple=False) + out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' -c foo.c" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1079,7 +1079,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags - out, ec = run_cmd("%s ld '' --enable-new-dtags foo.o" % script, simple=False) + out, ec = run_cmd("%s ld '' '%s/lib,%s/lib64,$ORIGIN' --enable-new-dtags foo.o" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) expected = '\n'.join([ "CMD_ARGS=('foo.o')", @@ -1097,7 +1097,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing no arguments - out, ec = run_cmd("%s gcc ''" % script, simple=False) + out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN'" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1108,7 +1108,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing a single empty argument - out, ec = run_cmd("%s ld.gold '' ''" % script, simple=False) + out, ec = run_cmd("%s ld.gold '' '%s/lib,%s/lib64,$ORIGIN' ''" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1120,7 +1120,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument - out, ec = run_cmd("%s gcc '' foo.c -L/foo -lfoo" % script, simple=False) + out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L/foo -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1135,7 +1135,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # relative paths passed to -L are *not* RPATH'ed in - out, ec = run_cmd("%s gcc '' foo.c -L../lib -lfoo" % script, simple=False) + out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L../lib -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1149,7 +1149,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, with value separated by a space - out, ec = run_cmd("%s gcc '' foo.c -L /foo -lfoo" % script, simple=False) + out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L /foo -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1164,7 +1164,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # multiple -L arguments, order should be preserved - out, ec = run_cmd("%s ld '' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % script, simple=False) + out, ec = run_cmd("%s ld '' '%s/lib,%s/lib64,$ORIGIN' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1186,7 +1186,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test specifying of custom rpath filter - out, ec = run_cmd("%s ld '/fo.*,/bar.*' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % script, simple=False) + out, ec = run_cmd("%s ld '/fo.*,/bar.*' '%s/lib,%s/lib64,$ORIGIN' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % (script, self.test_prefix, self.test_prefix), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1220,7 +1220,7 @@ def test_rpath_args_script(self): '-Wl,-rpath', '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) - out, ec = run_cmd("%s icc '' %s" % (script, args), simple=False) + out, ec = run_cmd("%s icc '' '%s/lib,%s/lib64,$ORIGIN' %s" % (script, self.test_prefix, self.test_prefix, args), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1261,7 +1261,7 @@ def test_rpath_args_script(self): '-o build/version.o', '../../gcc/version.c', ] - cmd = "%s g++ '' %s" % (script, ' '.join(args)) + cmd = "%s g++ '' '%s/lib,%s/lib64,$ORIGIN' %s" % (script, self.test_prefix, self.test_prefix, ' '.join(args)) out, ec = run_cmd(cmd, simple=False) self.assertEqual(ec, 0) From 2150c55dd33befa21c5f06081472b7d3ad1392a7 Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 30 Nov 2017 19:57:16 +0100 Subject: [PATCH 058/256] Include paths are now passed down from easyblock.py, where the installdir is defined. Shuffled tests to match the changed rpath mechanism. --- easybuild/scripts/rpath_args.py | 10 +++++++--- test/framework/toolchain.py | 5 +---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 8aab3d5d70..92cd61ceab 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -53,7 +53,10 @@ else: rpath_filter = None -rpath_include = rpath_include.split(',') +if not rpath_include: + rpath_include = [] +else: + rpath_include = rpath_include.split(',') version_mode = False cmd_args, cmd_args_rpath = [], [] @@ -110,8 +113,9 @@ cmd_args = cmd_args_rpath + cmd_args cmd_args_rpath = [] -for path in rpath_include: - cmd_args_rpath.append(flag_prefix + '-rpath=%s' % path) +if rpath_include: + for path in rpath_include: + cmd_args_rpath.append(flag_prefix + '-rpath=%s' % path) if not version_mode: diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 5189338a3a..4b8c199d55 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1284,7 +1284,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # verify that no -rpath arguments are injected when command is run in 'version check' mode - cmd = "%s g++ '' -v" % script + cmd = "%s g++ '' '%s/lib,%s/lib64,$ORIGIN' -v" % (script, self.test_prefix, self.test_prefix) out, ec = run_cmd(cmd, simple=False) self.assertEqual(ec, 0) self.assertEqual(out.strip(), "CMD_ARGS=('-v')") @@ -1336,9 +1336,6 @@ def test_toolchain_prepare_rpath(self): # no -rpath for /bar because of rpath filter out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""') expected = ' '.join([ - "'-Wl,-rpath=%s/lib'" % self.test_prefix, - "'-Wl,-rpath=%s/lib64'" % self.test_prefix, - "'-Wl,-rpath=$ORIGIN'", '-Wl,--disable-new-dtags', '-Wl,-rpath=/foo', '%(user)s.c', From 623aacbd43b64ff859a369489f37c6b7823933a6 Mon Sep 17 00:00:00 2001 From: John Donners Date: Wed, 6 Dec 2017 10:43:38 +0100 Subject: [PATCH 059/256] Changed accordingly --- easybuild/framework/easyblock.py | 3 ++- easybuild/scripts/rpath_args.py | 12 ++++-------- easybuild/tools/toolchain/toolchain.py | 9 +++------ test/framework/toolchain.py | 26 ++++++++++++++------------ 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0ed4c83910..3ec62bf8c0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1745,7 +1745,8 @@ def prepare_step(self, start_dir=True): self.rpath_include_dirs.append('$ORIGIN') # prepare toolchain: load toolchain module and dependencies, set up build environment - self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) + self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, + rpath_include_dirs=self.rpath_include_dirs) # handle allowed system dependencies for (name, version) in self.cfg['allow_system_deps']: diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 92cd61ceab..63500ec9ad 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -53,10 +53,10 @@ else: rpath_filter = None -if not rpath_include: - rpath_include = [] -else: +if rpath_include: rpath_include = rpath_include.split(',') +else: + rpath_include = [] version_mode = False cmd_args, cmd_args_rpath = [], [] @@ -112,11 +112,7 @@ # add -rpath flags in front cmd_args = cmd_args_rpath + cmd_args -cmd_args_rpath = [] -if rpath_include: - for path in rpath_include: - cmd_args_rpath.append(flag_prefix + '-rpath=%s' % path) - +cmds_args_rpath = [flag_prefix + '-rpath=%s' % inc for inc in rpath_include] if not version_mode: cmd_args = cmd_args_rpath + [ diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 93f7aa43bb..eb4bc420e2 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -797,11 +797,8 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) - # figure out list of patterns to use in rpath include - # rpath_include = build_option('rpath_include') - rpath_include = ','.join(['%s' % d for d in rpath_include_dirs or []]) - self.log.debug("Combined RPATH includes: '%s'", rpath_include) - + rpath_include = ','.join(rpath_include_dirs or []) + self.log.debug("Combined RPATH include paths: '%s'", rpath_include) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): @@ -831,8 +828,8 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, - 'rpath_include': rpath_include, 'rpath_filter': rpath_filter, + 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, } write_file(cmd_wrapper, cmd_wrapper_txt) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 4b8c199d55..ff4a972c00 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1065,8 +1065,10 @@ def test_rpath_args_script(self): """Test rpath_args.py script""" script = find_eb_script('rpath_args.py') + rpath_inc = '%(prefix)s/lib,%(prefix)s/lib64,$ORIGIN' % {'prefix': self.test_prefix} + # simplest possible compiler command - out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' -c foo.c" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1079,7 +1081,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags - out, ec = run_cmd("%s ld '' '%s/lib,%s/lib64,$ORIGIN' --enable-new-dtags foo.o" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) expected = '\n'.join([ "CMD_ARGS=('foo.o')", @@ -1097,7 +1099,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing no arguments - out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN'" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1108,7 +1110,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing a single empty argument - out, ec = run_cmd("%s ld.gold '' '%s/lib,%s/lib64,$ORIGIN' ''" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1120,7 +1122,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument - out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L/foo -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s gcc '' '%s' foo.c -L/foo -lfoo" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1135,7 +1137,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # relative paths passed to -L are *not* RPATH'ed in - out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L../lib -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1149,7 +1151,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, with value separated by a space - out, ec = run_cmd("%s gcc '' '%s/lib,%s/lib64,$ORIGIN' foo.c -L /foo -lfoo" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s gcc '' '%s' foo.c -L /foo -lfoo" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1164,7 +1166,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # multiple -L arguments, order should be preserved - out, ec = run_cmd("%s ld '' '%s/lib,%s/lib64,$ORIGIN' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s ld '' '%s' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1186,7 +1188,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test specifying of custom rpath filter - out, ec = run_cmd("%s ld '/fo.*,/bar.*' '%s/lib,%s/lib64,$ORIGIN' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % (script, self.test_prefix, self.test_prefix), simple=False) + out, ec = run_cmd("%s ld '/fo.*,/bar.*' '%s' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, @@ -1220,7 +1222,7 @@ def test_rpath_args_script(self): '-Wl,-rpath', '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) - out, ec = run_cmd("%s icc '' '%s/lib,%s/lib64,$ORIGIN' %s" % (script, self.test_prefix, self.test_prefix, args), simple=False) + out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False) self.assertEqual(ec, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -1261,7 +1263,7 @@ def test_rpath_args_script(self): '-o build/version.o', '../../gcc/version.c', ] - cmd = "%s g++ '' '%s/lib,%s/lib64,$ORIGIN' %s" % (script, self.test_prefix, self.test_prefix, ' '.join(args)) + cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args)) out, ec = run_cmd(cmd, simple=False) self.assertEqual(ec, 0) @@ -1284,7 +1286,7 @@ def test_rpath_args_script(self): self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # verify that no -rpath arguments are injected when command is run in 'version check' mode - cmd = "%s g++ '' '%s/lib,%s/lib64,$ORIGIN' -v" % (script, self.test_prefix, self.test_prefix) + cmd = "%s g++ '' '%s' -v" % (script, rpath_inc) out, ec = run_cmd(cmd, simple=False) self.assertEqual(ec, 0) self.assertEqual(out.strip(), "CMD_ARGS=('-v')") From 43dfdcf02eb1d4ff57db0a56c3802012ff436b8d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Dec 2017 19:25:39 +0100 Subject: [PATCH 060/256] fix remarks by @damianam --- easybuild/framework/easyblock.py | 23 +++------------- easybuild/tools/filetools.py | 1 - easybuild/tools/hooks.py | 47 ++++++++++++++++++++++---------- test/framework/hooks.py | 10 +++---- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8221198671..b608c1f888 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,7 +72,9 @@ from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file -from easybuild.tools.hooks import run_hook +from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP +from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP +from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -88,23 +90,6 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -BUILD_STEP = 'build' -CLEANUP_STEP = 'cleanup' -CONFIGURE_STEP = 'configure' -EXTENSIONS_STEP = 'extensions' -FETCH_STEP = 'fetch' -MODULE_STEP = 'module' -PACKAGE_STEP = 'package' -PATCH_STEP = 'patch' -PERMISSIONS_STEP = 'permissions' -POSTPROC_STEP = 'postproc' -PREPARE_STEP = 'prepare' -READY_STEP = 'ready' -SANITYCHECK_STEP = 'sanitycheck' -SOURCE_STEP = 'source' -TEST_STEP = 'test' -TESTCASES_STEP = 'testcases' - MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP] # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) @@ -2507,7 +2492,7 @@ def prepare_step_spec(initial): (False, lambda x: x.make_installdir), (True, lambda x: x.install_step), ] - install_step_spec = lambda initial: get_step('install', "installing", install_substeps, True, initial=initial) + install_step_spec = lambda init: get_step(INSTALL_STEP, "installing", install_substeps, True, initial=init) # format for step specifications: (stop_name: (description, list of functions, skippable)) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 19c8bd55c4..f80934f0ec 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -42,7 +42,6 @@ import fileinput import glob import hashlib -import imp import os import re import shutil diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index f604aaaaef..5ba4226e64 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -36,16 +36,35 @@ _log = fancylogger.getLogger('hooks', fname=False) -# this should be obtained via EasyBlock.get_steps(), -# but we can't import from easybuild.framework.easyblock without introducing a cyclic dependency... -STEP_NAMES = ['fetch', 'ready', 'source', 'patch', 'prepare', 'configure', 'build', 'test', 'install', 'extensions', - 'postproc', 'sanitycheck', 'cleanup', 'module', 'permissions', 'package', 'testcases'] +BUILD_STEP = 'build' +CLEANUP_STEP = 'cleanup' +CONFIGURE_STEP = 'configure' +EXTENSIONS_STEP = 'extensions' +FETCH_STEP = 'fetch' +INSTALL_STEP = 'install' +MODULE_STEP = 'module' +PACKAGE_STEP = 'package' +PATCH_STEP = 'patch' +PERMISSIONS_STEP = 'permissions' +POSTPROC_STEP = 'postproc' +PREPARE_STEP = 'prepare' +READY_STEP = 'ready' +SANITYCHECK_STEP = 'sanitycheck' +SOURCE_STEP = 'source' +TEST_STEP = 'test' +TESTCASES_STEP = 'testcases' + +# list of names for steps in installation procedure (in order of execution) +STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP, + INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, + PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] + KNOWN_HOOKS = ['%s_hook' % h for h in ['start'] + [p + '_' + s for s in STEP_NAMES for p in ['pre', 'post']] + ['end']] def load_hooks(hooks_path): """Load defined hooks (if any).""" - hooks = [] + hooks = {} if hooks_path: if not os.path.exists(hooks_path): @@ -61,10 +80,10 @@ def load_hooks(hooks_path): if attr.endswith('_hook'): hook = getattr(imported_hooks, attr) if callable(hook): - hooks.append(hook) + hooks.update({attr: hook}) else: _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) - _log.debug("Found hooks: %s", hooks) + _log.info("Found hooks: %s", sorted(hooks.keys())) except ImportError as err: raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err) else: @@ -80,15 +99,15 @@ def load_hooks(hooks_path): def verify_hooks(hooks): """Check whether list of obtained hooks only includes known hooks.""" unknown_hooks = [] - for hook in hooks: - if hook.__name__ not in KNOWN_HOOKS: - unknown_hooks.append(hook.__name__) + for key in hooks: + if key not in KNOWN_HOOKS: + unknown_hooks.append(key) if unknown_hooks: raise EasyBuildError("Found one or more unknown hooks: %s (known hooks: %s)", ', '.join(unknown_hooks), ', '.join(KNOWN_HOOKS)) else: - _log.info("Defined hooks verified, all known hooks: %s", ', '.join(h.__name__ for h in hooks)) + _log.info("Defined hooks verified, all known hooks: %s", ', '.join(h for h in hooks)) def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): @@ -111,10 +130,10 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): hook_name = hook_prefix + label + '_hook' - for hook in hooks: - if hook.__name__ == hook_name: + for key in hooks: + if key == hook_name: _log.info("Found %s hook", hook_name) - res = hook + res = hooks[key] break return res diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 10c0d242ea..626e9e4c6b 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -68,17 +68,17 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) self.assertEqual(len(hooks), 3) - self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook', 'start_hook']) - self.assertTrue(all(callable(h) for h in hooks)) + self.assertEqual(sorted(hooks.keys()), ['post_configure_hook', 'pre_install_hook', 'start_hook']) + self.assertTrue(all(callable(h) for h in hooks.values())) def test_find_hook(self): """Test for find_hook function.""" hooks = load_hooks(self.test_hooks_pymod) - post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0] - pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0] - start_hook = [h for h in hooks if h.__name__ == 'start_hook'][0] + post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0] + pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] + start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] self.assertEqual(find_hook('configure', hooks), None) self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None) From 7f336e1e9fc78644a3d382f5d0bc37a727d36034 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Dec 2017 19:45:16 +0100 Subject: [PATCH 061/256] make verify_hook issue suggestions for close matching hook names in error message for unknown hooks --- easybuild/tools/hooks.py | 15 +++++++++++++-- test/framework/hooks.py | 10 ++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 5ba4226e64..22ea8cdb74 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -27,6 +27,7 @@ :author: Kenneth Hoste (Ghent University) """ +import difflib import imp import os from vsc.utils import fancylogger @@ -104,8 +105,18 @@ def verify_hooks(hooks): unknown_hooks.append(key) if unknown_hooks: - raise EasyBuildError("Found one or more unknown hooks: %s (known hooks: %s)", - ', '.join(unknown_hooks), ', '.join(KNOWN_HOOKS)) + error_lines = ["Found one or more unknown hooks:"] + + for unknown_hook in unknown_hooks: + error_lines.append("* %s" % unknown_hook) + # try to find close match, may be just a typo in the hook name + close_matching_hooks = difflib.get_close_matches(unknown_hook, KNOWN_HOOKS, 2, 0.8) + if close_matching_hooks: + error_lines[-1] += " (did you mean %s?)" % ', or '.join("'%s'" % h for h in close_matching_hooks) + + error_lines.extend(['', "List of known hooks: %s" % ', '.join(KNOWN_HOOKS)]) + + raise EasyBuildError('\n'.join(error_lines)) else: _log.info("Defined hooks verified, all known hooks: %s", ', '.join(h for h in hooks)) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 626e9e4c6b..3cf6d92282 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -138,15 +138,21 @@ def test_verify_hooks(self): '', 'def there_is_no_such_hook():', ' pass', + 'def stat_hook(self):', + ' pass', 'def post_source_hook(self):', ' pass', - 'def another_faulty_hook(self):', + 'def install_hook(self):', ' pass', ]) write_file(test_broken_hooks_pymod, test_hooks_txt) - error_msg_pattern = "Found one or more unknown hooks: another_faulty_hook, there_is_no_such_hook \(known hooks:" + error_msg_pattern = r"Found one or more unknown hooks:\n" + error_msg_pattern += r"\* stat_hook \(did you mean 'start_hook'\?\)\n" + error_msg_pattern += r"\* there_is_no_such_hook\n" + error_msg_pattern += r"\* install_hook \(did you mean 'pre_install_hook', or 'post_install_hook'\?\)\n\n" + error_msg_pattern += r"List of known hooks: .*" self.assertErrorRegex(EasyBuildError, error_msg_pattern, load_hooks, test_broken_hooks_pymod) From c5d093ef660b7da8fb63bc5ac00299473510150e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Dec 2017 09:14:10 +0100 Subject: [PATCH 062/256] add support for --avail-hooks --- easybuild/tools/hooks.py | 2 +- easybuild/tools/options.py | 6 ++++++ test/framework/hooks.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 22ea8cdb74..65a84bf965 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -114,7 +114,7 @@ def verify_hooks(hooks): if close_matching_hooks: error_lines[-1] += " (did you mean %s?)" % ', or '.join("'%s'" % h for h in close_matching_hooks) - error_lines.extend(['', "List of known hooks: %s" % ', '.join(KNOWN_HOOKS)]) + error_lines.extend(['', "Run 'eb --avail-hooks' to get an overview of known hooks"]) raise EasyBuildError('\n'.join(error_lines)) else: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6db475fdcd..0eccd5aa35 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -73,6 +73,7 @@ from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, HAVE_GITHUB_API, HAVE_KEYRING from easybuild.tools.github import fetch_github_token +from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends from easybuild.tools.modules import avail_modules_tools @@ -514,6 +515,7 @@ def informative_options(self): 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs."), None, 'store_true', False), + 'avail-hooks': ("Show list of known hooks", None, 'store_true', False), 'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None), 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), @@ -728,6 +730,7 @@ def postprocess(self): self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, self.options.avail_toolchain_opts, + self.options.avail_hooks, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() @@ -909,6 +912,9 @@ def _postprocess_list_avail(self): if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() + if self.options.avail_hooks: + msg += self.avail_list('hooks (in order of execution)', KNOWN_HOOKS) + if self.options.unittest_file: self.log.info(msg) else: diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 3cf6d92282..993bcfa00d 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -152,7 +152,7 @@ def test_verify_hooks(self): error_msg_pattern += r"\* stat_hook \(did you mean 'start_hook'\?\)\n" error_msg_pattern += r"\* there_is_no_such_hook\n" error_msg_pattern += r"\* install_hook \(did you mean 'pre_install_hook', or 'post_install_hook'\?\)\n\n" - error_msg_pattern += r"List of known hooks: .*" + error_msg_pattern += r"Run 'eb --avail-hooks' to get an overview of known hooks" self.assertErrorRegex(EasyBuildError, error_msg_pattern, load_hooks, test_broken_hooks_pymod) From 9ce6d579b4d0eedfb3283465bb86d49470eed9ed Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 7 Dec 2017 10:17:48 +0100 Subject: [PATCH 063/256] Fixed typo --- easybuild/scripts/rpath_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 63500ec9ad..24136d4b4a 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -112,7 +112,7 @@ # add -rpath flags in front cmd_args = cmd_args_rpath + cmd_args -cmds_args_rpath = [flag_prefix + '-rpath=%s' % inc for inc in rpath_include] +cmd_args_rpath = [flag_prefix + '-rpath=%s' % inc for inc in rpath_include] if not version_mode: cmd_args = cmd_args_rpath + [ From 2d8ed12b351c127a5e181b389560869401f1a0d9 Mon Sep 17 00:00:00 2001 From: John Donners Date: Thu, 7 Dec 2017 10:25:19 +0100 Subject: [PATCH 064/256] Improved python code --- test/framework/toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index ff4a972c00..b336a77a86 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1085,8 +1085,8 @@ def test_rpath_args_script(self): self.assertEqual(ec, 0) expected = '\n'.join([ "CMD_ARGS=('foo.o')", - "RPATH_ARGS='--disable-new-dtags -rpath=%s/lib -rpath=%s/lib64 -rpath=$ORIGIN'" % (self.test_prefix, self.test_prefix), - '' + "RPATH_ARGS='--disable-new-dtags -rpath=%(prefix)s/lib -rpath=%(prefix)s/lib64 -rpath=$ORIGIN'" % + {'prefix': self.test_prefix},'' ]) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, From 34702010b270f397988bdde108025763e8991453 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 8 Dec 2017 17:39:15 +0100 Subject: [PATCH 065/256] enhance test_toy_rpath to make sure correct list of rpath filter paths is used --- test/framework/toy_build.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9427b247d0..a089927f21 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1644,10 +1644,40 @@ def test_toy_iter(self): def test_toy_rpath(self): """Test toy build using --rpath.""" - self.test_toy_build(extra_args=['--rpath', '--experimental'], raise_error=True) + def grab_gcc_rpath_wrapper_filter_arg(): + """Helper function to grab filter argument from last RPATH wrapper for 'gcc'.""" + rpath_wrappers_dir = glob.glob(os.path.join(os.getenv('TMPDIR'), '*', '*', 'rpath_wrappers'))[0] + gcc_rpath_wrapper_txt = read_file(os.path.join(rpath_wrappers_dir, 'gcc')) + + rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '([^ ]*)'.*", re.M) + res = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + + shutil.rmtree(rpath_wrappers_dir) + + return res.group(1) + + args = ['--rpath', '--experimental'] + self.test_toy_build(extra_args=args, raise_error=True) + + # by default, /lib and /usr are included in RPATH filter, + # together with temporary directory and build directory + rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + self.assertTrue('/lib.*' in rpath_filter_paths) + self.assertTrue('/usr.*' in rpath_filter_paths) + self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) + self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) # also test use of --rpath-filter - self.test_toy_build(extra_args=['--rpath', '--rpath-filter=/test.*,/foo.*', '--experimental'], raise_error=True) + args.extend(['--rpath-filter=/test.*,/foo/bar.*', '--disable-cleanup-tmpdir']) + outtxt = self.test_toy_build(extra_args=args, raise_error=True) + + # check whether rpath filter was set correctly + rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + self.assertTrue('/test.*' in rpath_filter_paths) + self.assertTrue('/foo/bar.*' in rpath_filter_paths) + self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) + self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) # test use of rpath toolchain option test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From eb5edab1f68e60eda3121754239bccd249138628 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 8 Dec 2017 17:43:36 +0100 Subject: [PATCH 066/256] re-instate $ORIGIN/../lib and $ORIGIN/../lib64 in RPATH locations injected by RPATH compiler wrapper --- easybuild/framework/easyblock.py | 8 +++++--- test/framework/toolchain.py | 35 ++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ec62bf8c0..230651dc0a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1737,12 +1737,14 @@ def prepare_step(self, start_dir=True): if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) - # always include self.installdir+'/lib' and self.installdir+'/lib64' and $ORIGIN + # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object # see also https://linux.die.net/man/8/ld-linux; - self.rpath_include_dirs.append(self.installdir+'/lib') - self.rpath_include_dirs.append(self.installdir+'/lib64') + self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib')) + self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64')) self.rpath_include_dirs.append('$ORIGIN') + self.rpath_include_dirs.append('$ORIGIN/../lib') + self.rpath_include_dirs.append('$ORIGIN/../lib64') # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index b336a77a86..3399a51dec 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1065,7 +1065,13 @@ def test_rpath_args_script(self): """Test rpath_args.py script""" script = find_eb_script('rpath_args.py') - rpath_inc = '%(prefix)s/lib,%(prefix)s/lib64,$ORIGIN' % {'prefix': self.test_prefix} + rpath_inc = ','.join([ + os.path.join(self.test_prefix, 'lib'), + os.path.join(self.test_prefix, 'lib64'), + '$ORIGIN', + '$ORIGIN/../lib', + '$ORIGIN/../lib64', + ]) # simplest possible compiler command out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) @@ -1074,6 +1080,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'-c'", "'foo.c'", @@ -1083,15 +1091,12 @@ def test_rpath_args_script(self): # linker command, --enable-new-dtags should be replaced with --disable-new-dtags out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) self.assertEqual(ec, 0) - expected = '\n'.join([ - "CMD_ARGS=('foo.o')", - "RPATH_ARGS='--disable-new-dtags -rpath=%(prefix)s/lib -rpath=%(prefix)s/lib64 -rpath=$ORIGIN'" % - {'prefix': self.test_prefix},'' - ]) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", "'--disable-new-dtags'", "'--disable-new-dtags'", "'foo.o'", @@ -1105,6 +1110,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", ] self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) @@ -1116,6 +1123,8 @@ def test_rpath_args_script(self): "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", "'--disable-new-dtags'", "''", ] @@ -1128,6 +1137,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/foo'", "'foo.c'", @@ -1143,6 +1154,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'foo.c'", "'-L../lib'", @@ -1157,6 +1170,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/foo'", "'foo.c'", @@ -1172,6 +1187,8 @@ def test_rpath_args_script(self): "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", "'--disable-new-dtags'", "'-rpath=/foo'", "'-rpath=/lib64'", @@ -1194,6 +1211,8 @@ def test_rpath_args_script(self): "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", "'--disable-new-dtags'", "'-rpath=/lib64'", "'-L/foo'", @@ -1228,6 +1247,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'-Wl,-rpath=/icc/lib/intel64'", "'-Wl,-rpath=/imkl/lib'", @@ -1271,6 +1292,8 @@ def test_rpath_args_script(self): "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", "'-DHAVE_CONFIG_H'", "'-I.'", From 3194f4f54b5a0afbd4b2f6c18feb2d0301e87bd1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 10 Dec 2017 11:26:46 +0100 Subject: [PATCH 067/256] add 'hooks' as last named argument to 'build_and_install_software' function --- easybuild/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 85537141c6..0cfda12e4b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -105,14 +105,14 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, hooks=None, exit_on_failure=True): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports - :param hooks: list of defined pre- and post-step hooks :param exit_on_failure: whether or not to exit on installation failure + :param hooks: list of defined pre- and post-step hooks """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since @@ -469,8 +469,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): exit_on_failure = not options.dump_test_report and not options.upload_test_report hooks = load_hooks(options.hooks) - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, hooks=hooks, - exit_on_failure=exit_on_failure) + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure, hooks=hooks) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] From ae50e00eb92fda1e3b7c9ba28c9d6dd6341d5bd4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 10 Dec 2017 12:00:12 +0100 Subject: [PATCH 068/256] add constants for 'pre_', 'post_' and '_hook' prefix/suffix of hook names --- easybuild/main.py | 8 ++++---- easybuild/tools/hooks.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 0cfda12e4b..d819906170 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -61,7 +61,7 @@ from easybuild.tools.filetools import adjust_permissions, cleanup, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr -from easybuild.tools.hooks import load_hooks, run_hook +from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs @@ -119,7 +119,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, ho # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) - run_hook('start', hooks) + run_hook(START, hooks) res = [] for ec in ecs: @@ -164,7 +164,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, ho res.append((ec, ec_res)) - run_hook('end', hooks) + run_hook(END, hooks) return res @@ -466,7 +466,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): - exit_on_failure = not options.dump_test_report and not options.upload_test_report + exit_on_failure = not (options.dump_test_report or options.upload_test_report) hooks = load_hooks(options.hooks) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 65a84bf965..05019a647a 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -55,12 +55,19 @@ TEST_STEP = 'test' TESTCASES_STEP = 'testcases' +START = 'start' +END = 'end' + +PRE_PREF = 'pre_' +POST_PREF = 'post_' +HOOK_SUFF = '_hook' + # list of names for steps in installation procedure (in order of execution) STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP, INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -KNOWN_HOOKS = ['%s_hook' % h for h in ['start'] + [p + '_' + s for s in STEP_NAMES for p in ['pre', 'post']] + ['end']] +KNOWN_HOOKS = [h + HOOK_SUFF for h in [START] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]] def load_hooks(hooks_path): @@ -78,7 +85,7 @@ def load_hooks(hooks_path): # import module that defines hooks, and collect all functions of which name ends with '_hook' imported_hooks = imp.load_source(hooks_filename, hooks_path) for attr in dir(imported_hooks): - if attr.endswith('_hook'): + if attr.endswith(HOOK_SUFF): hook = getattr(imported_hooks, attr) if callable(hook): hooks.update({attr: hook}) @@ -133,13 +140,13 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): res = None if pre_step_hook: - hook_prefix = 'pre_' + hook_prefix = PRE_PREF elif post_step_hook: - hook_prefix = 'post_' + hook_prefix = POST_PREF else: hook_prefix = '' - hook_name = hook_prefix + label + '_hook' + hook_name = hook_prefix + label + HOOK_SUFF for key in hooks: if key == hook_name: From 34410f32439f32da5a7eaa5c143b5632393967dc Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 11 Dec 2017 17:06:53 +0800 Subject: [PATCH 069/256] prepare release notes for eb350 --- RELEASE_NOTES | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 54879c477f..e9b04f7be1 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,29 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v3.5.0 (December 12th 2017) +--------------------------- + +feature release +- various enhancements, including: + - add framework support for foss-like toolchain with Spectrum MPI: gsolf (#2329) + - add support for --preview-pr (#2331) + - flesh out find_extension function, hoist dict with extract commands into a constant (#2336) + - add support for using self.start_dir rather than using self.cfg['start_dir'] (#2339) + - add support for 'exts_default_options' easyconfig parameter (#2345) + - allow use of `start_dir` in extensions (#2353) +- various bug fixes, including: + - fix typo in Class and comment, giolf != goolf. (#2327) + - fix minor issues with --inject-checksums (#2333) + - fix bug that slipped in with #2331 (#2337) + - fix error message when 'gv' Python package is not available (#2340) + - install paramiko version < 2.4.0 for Python 2.6 in Travis config (#2344) + - take exts_default_options easyconfig parameter into account for --inject-checksums (#2346) + - disable broken log rotation, avoid duplicate logging of output of executed commands under `--debug` (#2347) + - skip testing of --preview-pr if no GitHub token is available (#2348) + - use absolute paths to lib and lib64 icw $ORIGIN for the RPATH (#2358) + + v3.4.1 (October 17th 2017) -------------------------- From 38e43ba9ede217b1d763d7bc407c771b74547342 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 11 Dec 2017 16:33:06 +0100 Subject: [PATCH 070/256] Add giolfc toolchain. It uses gimpic, i.e. gcccuda + IntelMPI --- easybuild/toolchains/gimpic.py | 40 +++++++++++++++++++++++++++++++++ easybuild/toolchains/giolfc.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 easybuild/toolchains/gimpic.py create mode 100644 easybuild/toolchains/giolfc.py diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py new file mode 100644 index 0000000000..727f17f24f --- /dev/null +++ b/easybuild/toolchains/gimpic.py @@ -0,0 +1,40 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gimpic compiler toolchain (includes GCC and IntelMPI and CUDA). + +:author: Kenneth Hoste (Ghent University) +:author: Fotis Georgatos (Uni.Lu, NTUA) +:author: Ake Sandgren (HPC2N, Umea University) +""" + +from easybuild.toolchains.gcccuda import GccCUDA +from easybuild.toolchains.mpi.intelmpi import IntelMPI + + +class Gimpic(GccCUDA, IntelMPI): + """Compiler toolchain with GCC+CUDA and IntelMPI.""" + NAME = 'gimpic' + SUBTOOLCHAIN = GccCUDA.NAME diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py new file mode 100644 index 0000000000..d97de1ea05 --- /dev/null +++ b/easybuild/toolchains/giolfc.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for giolfc compiler toolchain (includes GCC+CUDA, IntelMPI, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +:author: Kenneth Hoste (Ghent University) +:author: Ake Sandgren (HPC2N, Umea University) +""" + +from easybuild.toolchains.gimpic import Gimpic +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + +class Giolfc(Gimpic, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC+CUDA, IntelMPI, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'giolfc' + SUBTOOLCHAIN = Gimpic.NAME + From 6ee37149e39faed85071372a6b668faa6de6d4a0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 12 Dec 2017 12:08:10 +0100 Subject: [PATCH 071/256] clean up release notes for EasyBuild v3.5.0 --- RELEASE_NOTES | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index e9b04f7be1..78aab815de 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -7,23 +7,22 @@ v3.5.0 (December 12th 2017) --------------------------- feature release +- add support for implementing pre- and post-step hooks (#2343) - various enhancements, including: - - add framework support for foss-like toolchain with Spectrum MPI: gsolf (#2329) - - add support for --preview-pr (#2331) + - add support for foss-like toolchain with Spectrum MPI: gsolf (#2329) + - add support for --preview-pr (#2331, #2337, #2348) + - see also http://easybuild.readthedocs.io/en/latest/Integration_with_GitHub.html#previewing-easyconfig-pull-requests-preview-pr - flesh out find_extension function, hoist dict with extract commands into a constant (#2336) - add support for using self.start_dir rather than using self.cfg['start_dir'] (#2339) - - add support for 'exts_default_options' easyconfig parameter (#2345) - - allow use of `start_dir` in extensions (#2353) + - add support for 'exts_default_options' easyconfig parameter (#2345, #2346) + - allow use of 'start_dir' easyconfig parameter in extensions (#2353) - various bug fixes, including: - - fix typo in Class and comment, giolf != goolf. (#2327) + - fix typo in 'giolf' toolchain definition (#2327) - fix minor issues with --inject-checksums (#2333) - - fix bug that slipped in with #2331 (#2337) - fix error message when 'gv' Python package is not available (#2340) - install paramiko version < 2.4.0 for Python 2.6 in Travis config (#2344) - - take exts_default_options easyconfig parameter into account for --inject-checksums (#2346) - - disable broken log rotation, avoid duplicate logging of output of executed commands under `--debug` (#2347) - - skip testing of --preview-pr if no GitHub token is available (#2348) - - use absolute paths to lib and lib64 icw $ORIGIN for the RPATH (#2358) + - disable broken log rotation, avoid duplicate logging of output of executed commands under '--debug' (#2347) + - also include $ORIGIN and absolute paths to 'lib' and 'lib64' subdirectories in RPATH locations (#2358) v3.4.1 (October 17th 2017) From 0d65079042e686e418bf840b5b208dc13f12363b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 12 Dec 2017 12:08:28 +0100 Subject: [PATCH 072/256] bump version to v3.5.0 --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d6ffdf0f81..9fa4369235 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.0.dev0') +VERSION = LooseVersion('3.5.0') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 0d85926f76dec3b21748146903b03db52eebc420 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 12 Dec 2017 12:10:05 +0100 Subject: [PATCH 073/256] include URLs for hooks docs --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 78aab815de..97c001580f 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -8,6 +8,7 @@ v3.5.0 (December 12th 2017) feature release - add support for implementing pre- and post-step hooks (#2343) + - documentation available at http://easybuild.readthedocs.io/en/latest/Hooks.html - various enhancements, including: - add support for foss-like toolchain with Spectrum MPI: gsolf (#2329) - add support for --preview-pr (#2331, #2337, #2348) From 77ea2e545575ca38b2e339d59a09c7ca062b68f1 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 13 Dec 2017 09:53:45 +0800 Subject: [PATCH 074/256] fix repo references in install develop script --- .../scripts/install-EasyBuild-develop.sh | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 42a9c5cc8f..b5cd7ddd28 100644 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -28,11 +28,20 @@ github_clone_branch() echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..." git clone --branch "${BRANCH}" "git@github.com:${GITHUB_USERNAME}/${REPO}.git" - echo "=== Adding and fetching HPC-UGent GitHub repository @ easybuilders/${REPO} ..." - cd "${REPO}" - git remote add "github_easybuilders" "git@github.com/easybuilders/${REPO}.git" - git fetch github_easybuilders - git branch --set-upstream "${BRANCH}" "github_easybuilders/${BRANCH}" + if [[ "$REPO" == "vsc"* ]] + then + echo "=== Adding and fetching HPC-UGent GitHub repository @ hpcugent/${REPO} ..." + cd "${REPO}" + git remote add "github_hpcugent" "git@github.com:hpcugent/${REPO}.git" + git fetch github_hpcugent + git branch --set-upstream "${BRANCH}" "github_hpcugent/${BRANCH}" + else + echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..." + cd "${REPO}" + git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git" + git fetch github_easybuilders + git branch --set-upstream "${BRANCH}" "github_easybuilders/${BRANCH}" + fi } # Print the content of the module @@ -116,7 +125,7 @@ github_clone_branch "easybuild-easyconfigs" "develop" github_clone_branch "easybuild" "develop" # Clone wiki repository with the 'master' branch -github_clone_branch "easybuild-wiki" "master" +#github_clone_branch "easybuild-wiki" "master" # Create the module file EB_DEVEL_MODULE_NAME="EasyBuild-develop" From a8a49bdd807943f848ecbcd269b36c1fc4bf34b6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Dec 2017 12:11:57 +0100 Subject: [PATCH 075/256] include #22360 in v3.5.0 release notes --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 97c001580f..754dc9f865 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -24,6 +24,7 @@ feature release - install paramiko version < 2.4.0 for Python 2.6 in Travis config (#2344) - disable broken log rotation, avoid duplicate logging of output of executed commands under '--debug' (#2347) - also include $ORIGIN and absolute paths to 'lib' and 'lib64' subdirectories in RPATH locations (#2358) + - fix repo references in install-EasyBuild-develop.sh script (#2360) v3.4.1 (October 17th 2017) From a72696b138ad15aefdfa7a3b7210b4629f760f58 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Dec 2017 09:31:18 +0100 Subject: [PATCH 076/256] fix release date for EasyBuild v3.5.0 --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 754dc9f865..f6993d89bc 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,7 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. -v3.5.0 (December 12th 2017) +v3.5.0 (December 15th 2017) --------------------------- feature release From f7c2f91810dad1506e8c5490df326cd1d8757b8b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Dec 2017 11:12:46 +0100 Subject: [PATCH 077/256] bump version to v3.5.1dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 9fa4369235..b244487272 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.0') +VERSION = LooseVersion('3.5.1.dev0') UNKNOWN = 'UNKNOWN' def get_git_revision(): From c1dfa4d6177f31c89f7da01c54619af210838b6c Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Wed, 20 Dec 2017 16:05:47 -0600 Subject: [PATCH 078/256] First attempt at support for new Environment Modules 4 --- easybuild/tools/modules.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 451e2a9aa7..2247ce0f02 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -144,6 +144,8 @@ class ModulesTool(object): VERSION_OPTION = '--version' # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) REQ_VERSION = None + # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) + MAX_VERSION = None # the regexp, should have a "version" group (multiline search) VERSION_REGEXP = None # modules tool user cache directory @@ -237,14 +239,30 @@ def set_and_check_version(self): except (OSError), err: raise EasyBuildError("Failed to check version: %s", err) - if self.REQ_VERSION is None: - self.log.debug("No version requirement defined.") + if self.REQ_VERSION is None and self.MAX_VERSION is None: + self.log.debug("No version requirement defined.") else: - if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): - raise EasyBuildError("EasyBuild requires v%s >= v%s (no rc), found v%s", - self.__class__.__name__, self.REQ_VERSION, self.version) + if self.REQ_VERSION is None: # MAX_VERSION is not None + self.log.debug("No required minimum version defined.") + if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): + raise EasyBuildError("EasyBuild requires v%s <= v%s (no rc), found v%s", + self.__class__.__name__, self.version, self.MAX_VERSION) + else: + self.log.debug('Version %s matches requirement <= %s' % (self.version, self.MAX_VERSION)) + elif self.MAX_VERSION is None: # REQ_VERSION is not None + self.log.debug("No allowed maximum version defined.") + if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): + raise EasyBuildError("EasyBuild requires v%s >= v%s (no rc), found v%s", + self.__class__.__name__, self.REQ_VERSION, self.version) + else: + self.log.debug('Version %s matches requirement >= %s' % (self.version, self.REQ_VERSION)) else: - self.log.debug('Version %s matches requirement %s' % (self.version, self.REQ_VERSION)) + self.log.debug("Required minimum version and maximum allowed version defined.") + # Check that we have not made a mistake and set REQ_VERSION > MAX_VERSION + if StrictVersion(self.REQ_VERSION) > StrictVersion(self.MAX_VERSION): + raise EasyBuildError("v%s, REQ_VERSION must be <= MAX_VERSION", self.__class__) + else: + self.log.debug('Version %s matches requirements >= %s, <= %s' % (self.version, self.REQ_VERSION, self.MAX_VERSION)) MODULE_VERSION_CACHE[self.COMMAND] = self.version @@ -997,12 +1015,18 @@ class EnvironmentModulesC(ModulesTool): """Interface to (C) environment modules (modulecmd).""" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' + MAX_VERSION = '3.2.10' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def update(self): """Update after new modules were added.""" pass +class EnvironmentModules4(EnvironmentModulesC): + """Interface to environment modules 4+.""" + REQ_VERSION = '4.0.0' + MAX_VERSION = None + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' class EnvironmentModulesTcl(EnvironmentModulesC): """Interface to (Tcl) environment modules (modulecmd.tcl).""" @@ -1014,6 +1038,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None + MAX_VERSION = '1.923' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From 9cd2c63cb6823c27b1b28cf7f3072561adc69277 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Thu, 28 Dec 2017 12:20:32 -0600 Subject: [PATCH 079/256] Corrected logic around new MAX_VERSION setting for ModulesTool. Removed the hastily-named EnvironmentModules4 class until there is a better implementation. Style corrections. --- easybuild/tools/modules.py | 46 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 2247ce0f02..8399ea15f6 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -240,29 +240,23 @@ def set_and_check_version(self): raise EasyBuildError("Failed to check version: %s", err) if self.REQ_VERSION is None and self.MAX_VERSION is None: - self.log.debug("No version requirement defined.") - else: - if self.REQ_VERSION is None: # MAX_VERSION is not None - self.log.debug("No required minimum version defined.") - if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): - raise EasyBuildError("EasyBuild requires v%s <= v%s (no rc), found v%s", - self.__class__.__name__, self.version, self.MAX_VERSION) - else: - self.log.debug('Version %s matches requirement <= %s' % (self.version, self.MAX_VERSION)) - elif self.MAX_VERSION is None: # REQ_VERSION is not None - self.log.debug("No allowed maximum version defined.") - if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): - raise EasyBuildError("EasyBuild requires v%s >= v%s (no rc), found v%s", - self.__class__.__name__, self.REQ_VERSION, self.version) - else: - self.log.debug('Version %s matches requirement >= %s' % (self.version, self.REQ_VERSION)) + self.log.debug("No version requirement defined.") + + if self.REQ_VERSION is not None: + self.log.debug("Required minimum version defined.") + if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): + raise EasyBuildError("EasyBuild requires v%s >= v%s, found v%s", + self.__class__.__name__, self.version, self.REQ_VERSION) else: - self.log.debug("Required minimum version and maximum allowed version defined.") - # Check that we have not made a mistake and set REQ_VERSION > MAX_VERSION - if StrictVersion(self.REQ_VERSION) > StrictVersion(self.MAX_VERSION): - raise EasyBuildError("v%s, REQ_VERSION must be <= MAX_VERSION", self.__class__) - else: - self.log.debug('Version %s matches requirements >= %s, <= %s' % (self.version, self.REQ_VERSION, self.MAX_VERSION)) + self.log.debug('Version %s matches requirement >= %s', self.version, self.REQ_VERSION) + + if self.MAX_VERSION is not None: + self.log.debug("Maximum allowed version defined.") + if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): + raise EasyBuildError("EasyBuild requires v%s <= v%s, found v%s", + self.__class__.__name__, self.version, self.MAX_VERSION) + else: + self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION) MODULE_VERSION_CACHE[self.COMMAND] = self.version @@ -1015,19 +1009,13 @@ class EnvironmentModulesC(ModulesTool): """Interface to (C) environment modules (modulecmd).""" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' - MAX_VERSION = '3.2.10' + MAX_VERSION = '3.99' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def update(self): """Update after new modules were added.""" pass -class EnvironmentModules4(EnvironmentModulesC): - """Interface to environment modules 4+.""" - REQ_VERSION = '4.0.0' - MAX_VERSION = None - VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' - class EnvironmentModulesTcl(EnvironmentModulesC): """Interface to (Tcl) environment modules (modulecmd.tcl).""" # Tcl environment modules have no --terse (yet), From cda2bd29b3f2a98739a11c29e764fb9fd587d71a Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Thu, 28 Dec 2017 12:27:13 -0600 Subject: [PATCH 080/256] Removed superfluous MAX_VERSION from EnvironmentModulesTcl --- easybuild/tools/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8399ea15f6..2137cc09d8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1026,7 +1026,6 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None - MAX_VERSION = '1.923' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From 26f414a4177edf81d3eba50a8ff387ec52e46cda Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Thu, 28 Dec 2017 13:07:49 -0600 Subject: [PATCH 081/256] Added new class EnvironmentModules(EnvironmentModulesTcl) to support EnvironmentModules 4.0+ --- easybuild/tools/modules.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 2137cc09d8..31eb1cef26 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1088,6 +1088,13 @@ def remove_module_path(self, path, set_mod_paths=True): self.set_mod_paths() +class EnvironmentModules(EnvironmentModulesTcl): + """Interface to environment modules 4.0+""" + COMMAND = os.environ['MODULESHOME'] + '/libexec/modulecmd.tcl' + REQ_VERSION = '4.0.0' + MAX_VERSION = None + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + class Lmod(ModulesTool): """Interface to Lmod.""" COMMAND = 'lmod' From 52c5bcb393a7e8f0a86364177cf9e47bcec52eca Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Thu, 28 Dec 2017 15:05:29 -0600 Subject: [PATCH 082/256] In EvironmentModules class, use os.path.join when setting COMMAND instead of string concatenation. Add a second blank line after EnvironmentModules class to match style of other classes. --- easybuild/tools/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 31eb1cef26..c0b37a0144 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1090,11 +1090,12 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - COMMAND = os.environ['MODULESHOME'] + '/libexec/modulecmd.tcl' + COMMAND = os.path.join(os.environ['MODULESHOME'], 'libexec', 'modulecmd.tcl') REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' - + + class Lmod(ModulesTool): """Interface to Lmod.""" COMMAND = 'lmod' From 61055f6b895f97718e9e7c40a8eff41b2b12dfdf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2017 10:31:27 +0100 Subject: [PATCH 083/256] install SQLAlchemy < 1.2.0 with Python 2.6 in Travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 64b84d8962..2b92582f92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ before_install: - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install pydot==1.1.0; else pip install pydot; fi # paramiko (dep for GC3Pie) 2.4.0 & more recent doesn't work with Python 2.6 - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'paramiko<2.4.0'; else pip install paramiko; fi + # SQLAlchemy (dep for GC3Pie) 1.2.0 & more recent doesn't work with Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'SQLAlchemy<1.2.0'; else pip install SQLAlchemy; fi # optional Python packages for EasyBuild - pip install autopep8 GC3Pie pycodestyle python-graph-dot python-hglib PyYAML # git config is required to make actual git commits (cfr. tests for GitRepository) From 6c6bb6165483a36f90fd453041193953ed3d1575 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2017 10:25:08 +0100 Subject: [PATCH 084/256] also test against Modules v4 --- .travis.yml | 5 ++++- easybuild/scripts/install_eb_dep.sh | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 64b84d8962..c03baa4f3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - LMOD_VERSION=7.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules matrix: # mark build as finished as soon as job has failed fast_finish: true @@ -44,6 +45,8 @@ before_install: - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install pydot==1.1.0; else pip install pydot; fi # paramiko (dep for GC3Pie) 2.4.0 & more recent doesn't work with Python 2.6 - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'paramiko<2.4.0'; else pip install paramiko; fi + # SQLAlchemy (dep for GC3Pie) 1.2.0 & more recent doesn't work with Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'SQLAlchemy<1.2.0'; else pip install SQLAlchemy; fi # optional Python packages for EasyBuild - pip install autopep8 GC3Pie pycodestyle python-graph-dot python-hglib PyYAML # git config is required to make actual git commits (cfr. tests for GitRepository) @@ -62,7 +65,7 @@ script: # make sure 'ml' alias is defined, otherwise sourcing the init script fails (silently) for Lmod (< 5.9.3) - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi # set up environment for modules tool (if $MOD_INIT is defined) - - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; fi + - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi # install GitHub token - if [ ! -z $GITHUB_TOKEN ]; then if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh index b9694f9dbb..4610e7626b 100755 --- a/easybuild/scripts/install_eb_dep.sh +++ b/easybuild/scripts/install_eb_dep.sh @@ -15,12 +15,17 @@ PKG_VERSION=`echo $PKG | sed 's/.*-//g'` CONFIG_OPTIONS= PRECONFIG_CMD= -if [ x$PKG_NAME == 'xmodules' ]; then +if [ x$PKG_NAME == 'xmodules' ] && [ x$PKG_VERSION == 'x3.2.10' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" BACKUP_PKG_URL="https://easybuilders.github.io/easybuild/files/${PKG}.tar.gz" export PATH=$PREFIX/Modules/$PKG_VERSION/bin:$PATH export MOD_INIT=$HOME/Modules/$PKG_VERSION/init/bash +elif [ x$PKG_NAME == 'xmodules' ]; then + PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" + export PATH=$PREFIX/bin:$PATH + export MOD_INIT=$HOME/init/bash + elif [ x$PKG_NAME == 'xlua' ]; then PKG_URL="http://downloads.sourceforge.net/project/lmod/${PKG}.tar.gz" BACKUP_PKG_URL="https://easybuilders.github.io/easybuild/files/${PKG}.tar.gz" @@ -55,7 +60,7 @@ fi set -e # environment-modules needs a patch to work with Tcl8.6 -if [ x$PKG_NAME == 'xmodules' ]; then +if [ x$PKG_NAME == 'xmodules' ] && [ x$PKG_VERSION == 'x3.2.10' ]; then wget -O 'modules-tcl8.6.patch' 'https://easybuilders.github.io/easybuild/files/modules-3.2.10-tcl8.6.patch' patch ${PKG}/cmdModule.c modules-tcl8.6.patch fi From 656f37fd44f4ae616f0acca2038627d4c16ee37a Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 09:38:43 -0600 Subject: [PATCH 085/256] When setting COMMAND in EnvironmentModules class, use os.getenv instead of os.environment when getting value of MODULESHOME. The latter results in a traceback if MODULESHOME is undefined. --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c0b37a0144..74442699b0 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1090,7 +1090,7 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - COMMAND = os.path.join(os.environ['MODULESHOME'], 'libexec', 'modulecmd.tcl') + COMMAND = os.path.join(os.getenv('MODULESHOME'), 'libexec', 'modulecmd.tcl') REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' From 29dfe3d2411a730c687732a08318b8d5d3c17099 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 10:35:45 -0600 Subject: [PATCH 086/256] When setting COMMAND in EnvironmentModules class, provide a fallback value in case MODULESHOME is undefined --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 74442699b0..664ec84d6a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1090,7 +1090,7 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - COMMAND = os.path.join(os.getenv('MODULESHOME'), 'libexec', 'modulecmd.tcl') + COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' From 585f1eec6bd484740d4bba66c567d29febed59f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2017 17:50:07 +0100 Subject: [PATCH 087/256] don't pass empty argument to 'module purge' --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 664ec84d6a..4b1289d279 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -542,7 +542,7 @@ def purge(self): Purge loaded modules. """ self.log.debug("List of loaded modules before purge: %s" % os.getenv('_LMFILES_')) - self.run_module('purge', '') + self.run_module('purge') def show(self, mod_name): """ From 763a2e0b25f0c3a4960cbc41ea318033ca2e0d0f Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 13:51:05 -0600 Subject: [PATCH 088/256] In purge method of ModulesTool class, do not pass empty argument to module purge. It causes an extraneous argument error with Modules 4+ --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 664ec84d6a..4b1289d279 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -542,7 +542,7 @@ def purge(self): Purge loaded modules. """ self.log.debug("List of loaded modules before purge: %s" % os.getenv('_LMFILES_')) - self.run_module('purge', '') + self.run_module('purge') def show(self, mod_name): """ From 1c85daa2a68766ee00c515500193c8caecaf8369 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 14:31:16 -0600 Subject: [PATCH 089/256] In test_avail method of ModulesTest class, handle EnvironmentModules (>=4.0) like EnvironmentModulesTcl (<=1.293) --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 780c88aa85..ca0f3343f4 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -99,7 +99,7 @@ def test_avail(self): ms = self.modtool.available('GCC') expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/4.7.2'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C does prefix matching - if not isinstance(self.modtool, EnvironmentModulesTcl): + if not isinstance(self.modtool, EnvironmentModulesTcl) or not isinstance(self.modtool, EnvironmentModules): expected.append('GCCcore/6.2.0') self.assertEqual(ms, expected) From c59565ee00c8b0bb95f9b583dbd0330b51ae5e46 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 14:35:24 -0600 Subject: [PATCH 090/256] In test_avail method of ModulesTest class, handle EnvironmentModules (>=4.0) like EnvironmentModulesTcl (<=1.293) --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index ca0f3343f4..44f088981c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -99,7 +99,7 @@ def test_avail(self): ms = self.modtool.available('GCC') expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/4.7.2'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C does prefix matching - if not isinstance(self.modtool, EnvironmentModulesTcl) or not isinstance(self.modtool, EnvironmentModules): + if not isinstance(self.modtool, EnvironmentModulesTcl) and not isinstance(self.modtool, EnvironmentModules): expected.append('GCCcore/6.2.0') self.assertEqual(ms, expected) From 6b6588bf0a39ec632212c27a2cbdd26d32731f45 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 14:52:22 -0600 Subject: [PATCH 091/256] import EnvironmentModules class --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 44f088981c..2979243f02 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -45,7 +45,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, write_file -from easybuild.tools.modules import EnvironmentModulesTcl, Lmod +from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesTcl, Lmod from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches from easybuild.tools.run import run_cmd From 2ce48cce2a28839e1221dd800fdbd73b4949ff44 Mon Sep 17 00:00:00 2001 From: ebagrenrut Date: Fri, 29 Dec 2017 16:01:18 -0600 Subject: [PATCH 092/256] EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to ModulesC, so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules --- test/framework/modules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 2979243f02..1f86a017cd 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -99,7 +99,8 @@ def test_avail(self): ms = self.modtool.available('GCC') expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/4.7.2'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C does prefix matching - if not isinstance(self.modtool, EnvironmentModulesTcl) and not isinstance(self.modtool, EnvironmentModules): + # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to ModulesC, so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules + if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): expected.append('GCCcore/6.2.0') self.assertEqual(ms, expected) From 0b7d344c0cb5efa4c9eb1bfe8a742359c5c02bf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 30 Dec 2017 11:27:55 +0100 Subject: [PATCH 093/256] update test_load, Modules v4.0 does not reload already loaded modules (+ break up long comment) --- test/framework/modules.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 1f86a017cd..9281aec16a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -98,8 +98,9 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/4.7.2'] - # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C does prefix matching - # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to ModulesC, so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules + # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching + # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., + # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): expected.append('GCCcore/6.2.0') self.assertEqual(ms, expected) @@ -192,13 +193,19 @@ def test_load(self): # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last self.modtool.load(['GCC/4.6.4']) - self.assertTrue(os.environ.get('EBROOTGCC')) + + # environment modules v4.0 does not reload already loaded modules, will be changed in v4.2 + modtool_ver = StrictVersion(self.modtool.version) + if not isinstance(self.modtool, EnvironmentModules) or modtool_ver >= StrictVersion('4.2'): + self.assertTrue(os.environ.get('EBROOTGCC')) + if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod self.assertTrue(self.modtool.loaded_modules()[-1] == 'GCC/4.6.4') # set things up for checking that GCC does *not* get reloaded when requested - del os.environ['EBROOTGCC'] + if 'EBROOTGCC' in os.environ: + del os.environ['EBROOTGCC'] self.modtool.load(['OpenMPI/1.6.4-GCC-4.6.4']) if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod From 792e1340db62a84d769d259ab02c503d70068ded Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2018 22:05:33 +0100 Subject: [PATCH 094/256] disable testing of bootstrap script when Modules v4 is used as modules tool --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c03baa4f3f..18ad417018 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,8 +96,12 @@ script: - EB_BOOTSTRAP_EXPECTED="20170808.01 b78eac49374b26c78a16ea41efcde8c03d21649087f650e08f5339e8e48d7c6e" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap + - if [ "x$TEST_EASYBUILD_MODULES_TOOL" != 'xEnvironmentModules' ]; then + python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; + fi; # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH # simply sanity check on bootstrapped EasyBuild module - - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version + - if [ "x$TEST_EASYBUILD_MODULES_TOOL" != 'xEnvironmentModules' ]; then + module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version; + fi; From 7a302c047e10b578d3dd94fc20a63518a6ee6bc8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jan 2018 14:26:47 +0100 Subject: [PATCH 095/256] bump copyright to 2018 in license headers --- .travis.yml | 2 +- easybuild/__init__.py | 2 +- easybuild/framework/__init__.py | 2 +- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/__init__.py | 2 +- easybuild/framework/easyconfig/constants.py | 2 +- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/__init__.py | 2 +- easybuild/framework/easyconfig/format/convert.py | 2 +- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 2 +- easybuild/framework/easyconfig/format/pyheaderconfigobj.py | 2 +- easybuild/framework/easyconfig/format/two.py | 2 +- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/licenses.py | 2 +- easybuild/framework/easyconfig/parser.py | 2 +- easybuild/framework/easyconfig/style.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/framework/easyconfig/types.py | 2 +- easybuild/framework/extension.py | 2 +- easybuild/framework/extensioneasyblock.py | 2 +- easybuild/main.py | 2 +- easybuild/scripts/add_header.py | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- easybuild/scripts/fix_broken_easyconfigs.py | 2 +- easybuild/scripts/fix_docs.py | 2 +- easybuild/scripts/generate_software_list.py | 2 +- easybuild/scripts/mk_tmpl_easyblock_for.py | 2 +- easybuild/scripts/port_easyblock.py | 2 +- easybuild/scripts/prep_for_release.py | 2 +- easybuild/scripts/repo_setup.py | 2 +- easybuild/scripts/rpath_args.py | 2 +- easybuild/toolchains/__init__.py | 2 +- easybuild/toolchains/cgmpich.py | 2 +- easybuild/toolchains/cgmpolf.py | 2 +- easybuild/toolchains/cgmvapich2.py | 2 +- easybuild/toolchains/cgmvolf.py | 2 +- easybuild/toolchains/cgompi.py | 2 +- easybuild/toolchains/cgoolf.py | 2 +- easybuild/toolchains/clanggcc.py | 2 +- easybuild/toolchains/compiler/__init__.py | 2 +- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/craype.py | 2 +- easybuild/toolchains/compiler/cuda.py | 2 +- easybuild/toolchains/compiler/dummycompiler.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- easybuild/toolchains/compiler/inteliccifort.py | 2 +- easybuild/toolchains/craycce.py | 2 +- easybuild/toolchains/craygnu.py | 2 +- easybuild/toolchains/crayintel.py | 2 +- easybuild/toolchains/craypgi.py | 2 +- easybuild/toolchains/dummy.py | 2 +- easybuild/toolchains/fft/__init__.py | 2 +- easybuild/toolchains/fft/fftw.py | 2 +- easybuild/toolchains/fft/intelfftw.py | 2 +- easybuild/toolchains/foss.py | 2 +- easybuild/toolchains/gcc.py | 2 +- easybuild/toolchains/gcccore.py | 2 +- easybuild/toolchains/gcccuda.py | 2 +- easybuild/toolchains/gimkl.py | 2 +- easybuild/toolchains/gimpi.py | 2 +- easybuild/toolchains/gimpic.py | 2 +- easybuild/toolchains/giolf.py | 2 +- easybuild/toolchains/giolfc.py | 2 +- easybuild/toolchains/gmacml.py | 2 +- easybuild/toolchains/gmpich.py | 2 +- easybuild/toolchains/gmpich2.py | 2 +- easybuild/toolchains/gmpolf.py | 2 +- easybuild/toolchains/gmvapich2.py | 2 +- easybuild/toolchains/gmvolf.py | 2 +- easybuild/toolchains/gnu.py | 2 +- easybuild/toolchains/goalf.py | 2 +- easybuild/toolchains/gompi.py | 2 +- easybuild/toolchains/gompic.py | 2 +- easybuild/toolchains/goolf.py | 2 +- easybuild/toolchains/goolfc.py | 2 +- easybuild/toolchains/gpsmpi.py | 2 +- easybuild/toolchains/gpsolf.py | 2 +- easybuild/toolchains/gqacml.py | 2 +- easybuild/toolchains/gsmpi.py | 2 +- easybuild/toolchains/gsolf.py | 2 +- easybuild/toolchains/iccifort.py | 2 +- easybuild/toolchains/iccifortcuda.py | 2 +- easybuild/toolchains/ictce.py | 2 +- easybuild/toolchains/iimkl.py | 2 +- easybuild/toolchains/iimpi.py | 2 +- easybuild/toolchains/iimpic.py | 2 +- easybuild/toolchains/iiqmpi.py | 2 +- easybuild/toolchains/impich.py | 2 +- easybuild/toolchains/impmkl.py | 2 +- easybuild/toolchains/intel-para.py | 2 +- easybuild/toolchains/intel.py | 2 +- easybuild/toolchains/intelcuda.py | 2 +- easybuild/toolchains/iomkl.py | 2 +- easybuild/toolchains/iompi.py | 2 +- easybuild/toolchains/ipsmpi.py | 2 +- easybuild/toolchains/iqacml.py | 2 +- easybuild/toolchains/ismkl.py | 2 +- easybuild/toolchains/linalg/__init__.py | 2 +- easybuild/toolchains/linalg/acml.py | 2 +- easybuild/toolchains/linalg/atlas.py | 2 +- easybuild/toolchains/linalg/blacs.py | 2 +- easybuild/toolchains/linalg/flame.py | 2 +- easybuild/toolchains/linalg/gotoblas.py | 2 +- easybuild/toolchains/linalg/intelmkl.py | 2 +- easybuild/toolchains/linalg/lapack.py | 2 +- easybuild/toolchains/linalg/libsci.py | 2 +- easybuild/toolchains/linalg/openblas.py | 2 +- easybuild/toolchains/linalg/scalapack.py | 2 +- easybuild/toolchains/mpi/__init__.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/intelmpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpich2.py | 2 +- easybuild/toolchains/mpi/mvapich2.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/toolchains/mpi/psmpi.py | 2 +- easybuild/toolchains/mpi/qlogicmpi.py | 2 +- easybuild/toolchains/mpi/spectrummpi.py | 2 +- easybuild/toolchains/pomkl.py | 2 +- easybuild/toolchains/pompi.py | 2 +- easybuild/tools/__init__.py | 2 +- easybuild/tools/asyncprocess.py | 2 +- easybuild/tools/build_details.py | 2 +- easybuild/tools/build_log.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/convert.py | 2 +- easybuild/tools/deprecated/__init__.py | 2 +- easybuild/tools/docs.py | 2 +- easybuild/tools/environment.py | 2 +- easybuild/tools/filetools.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/hooks.py | 2 +- easybuild/tools/include.py | 2 +- easybuild/tools/jenkins.py | 2 +- easybuild/tools/job/__init__.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/gc3pie.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/module_naming_scheme/__init__.py | 2 +- easybuild/tools/module_naming_scheme/categorized_mns.py | 2 +- easybuild/tools/module_naming_scheme/easybuild_mns.py | 2 +- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 2 +- easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py | 2 +- easybuild/tools/module_naming_scheme/mns.py | 2 +- easybuild/tools/module_naming_scheme/toolchain.py | 2 +- easybuild/tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/multidiff.py | 2 +- easybuild/tools/options.py | 2 +- easybuild/tools/package/__init__.py | 2 +- easybuild/tools/package/package_naming_scheme/__init__.py | 2 +- easybuild/tools/package/package_naming_scheme/easybuild_pns.py | 2 +- easybuild/tools/package/package_naming_scheme/pns.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/parallelbuild.py | 2 +- easybuild/tools/repository/__init__.py | 2 +- easybuild/tools/repository/filerepo.py | 2 +- easybuild/tools/repository/gitrepo.py | 2 +- easybuild/tools/repository/hgrepo.py | 2 +- easybuild/tools/repository/repository.py | 2 +- easybuild/tools/repository/svnrepo.py | 2 +- easybuild/tools/robot.py | 2 +- easybuild/tools/run.py | 2 +- easybuild/tools/systemtools.py | 2 +- easybuild/tools/testing.py | 2 +- easybuild/tools/toolchain/__init__.py | 2 +- easybuild/tools/toolchain/compiler.py | 2 +- easybuild/tools/toolchain/constants.py | 2 +- easybuild/tools/toolchain/fft.py | 2 +- easybuild/tools/toolchain/linalg.py | 2 +- easybuild/tools/toolchain/mpi.py | 2 +- easybuild/tools/toolchain/options.py | 2 +- easybuild/tools/toolchain/toolchain.py | 2 +- easybuild/tools/toolchain/toolchainvariables.py | 2 +- easybuild/tools/toolchain/utilities.py | 2 +- easybuild/tools/toolchain/variables.py | 2 +- easybuild/tools/utilities.py | 2 +- easybuild/tools/variables.py | 2 +- easybuild/tools/version.py | 2 +- setup.py | 2 +- test/__init__.py | 2 +- test/framework/__init__.py | 2 +- test/framework/asyncprocess.py | 2 +- test/framework/build_log.py | 2 +- test/framework/config.py | 2 +- test/framework/docs.py | 2 +- test/framework/easyblock.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/easyconfigformat.py | 2 +- test/framework/easyconfigparser.py | 2 +- test/framework/easyconfigversion.py | 2 +- test/framework/ebconfigobj.py | 2 +- test/framework/environment.py | 2 +- test/framework/filetools.py | 2 +- test/framework/format_convert.py | 2 +- test/framework/general.py | 2 +- test/framework/github.py | 2 +- test/framework/hooks.py | 2 +- test/framework/include.py | 2 +- test/framework/license.py | 2 +- test/framework/module_generator.py | 2 +- test/framework/modules.py | 2 +- test/framework/modulestool.py | 2 +- test/framework/options.py | 2 +- test/framework/package.py | 2 +- test/framework/parallelbuild.py | 2 +- test/framework/repository.py | 2 +- test/framework/robot.py | 2 +- test/framework/run.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foofoo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/g/gcc.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/bar.py | 2 +- .../sandbox/easybuild/easyblocks/generic/configuremake.py | 2 +- .../sandbox/easybuild/easyblocks/generic/dummyextension.py | 2 +- .../framework/sandbox/easybuild/easyblocks/generic/toolchain.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/h/hpl.py | 2 +- test/framework/sandbox/easybuild/easyblocks/s/scalapack.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/sandbox/easybuild/tools/__init__.py | 2 +- .../sandbox/easybuild/tools/module_naming_scheme/__init__.py | 2 +- .../tools/module_naming_scheme/broken_module_naming_scheme.py | 2 +- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- .../module_naming_scheme/test_module_naming_scheme_more.py | 2 +- test/framework/scripts.py | 2 +- test/framework/style.py | 2 +- test/framework/suite.py | 2 +- test/framework/systemtools.py | 2 +- test/framework/toolchain.py | 2 +- test/framework/toolchainvariables.py | 2 +- test/framework/toy_build.py | 2 +- test/framework/tweak.py | 2 +- test/framework/type_checking.py | 2 +- test/framework/utilities.py | 2 +- test/framework/variables.py | 2 +- test/framework/yeb.py | 2 +- 244 files changed, 244 insertions(+), 244 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b92582f92..75a1fae86b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,7 +92,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20170808.01 b78eac49374b26c78a16ea41efcde8c03d21649087f650e08f5339e8e48d7c6e" + - EB_BOOTSTRAP_EXPECTED="20170808.01 7a22faf5ee9d249807fd883e435008fb84c11129425ec09d3cc76918b36ef3d5" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 32eb31eacf..12c8c6ff23 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index 2dca6bc36d..1da7c9b0a1 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8fe28fafe..806061fe84 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 084fe75bfb..091730390f 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index e2b6f564d0..9752e206f9 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 864985b1d0..d45c3c81fc 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ada1045e19..b62a5c5430 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index 981b7fab02..4440c48efc 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 5b7fdc577d..66cd83c31c 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index b035c555d1..0d4b68fc30 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index aca516037f..e33c245aa6 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 9ba48774f1..a643069edd 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index f7f2a6301c..db8b2cf65c 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index 0b18723cea..a8e2c446ad 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 750bdcfca3..54f987ccad 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index d1fa2fbfc8..7e8853be19 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index bf1e2c610b..0f1bfa8bca 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index a8c3183dcc..fc325550f7 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2017 Ghent University +# Copyright 2016-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 6be4af2ad7..0793d979ef 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index bb63cc2758..c0460c3c71 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f91b695116..7c50715112 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index a6cf4ad831..8ba9458269 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 9af9b1b8df..c6ff2c0b8e 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 67e497a100..26952cc175 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/main.py b/easybuild/main.py index d819906170..02345fb867 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/add_header.py b/easybuild/scripts/add_header.py index bf5adbb136..2f260abe11 100644 --- a/easybuild/scripts/add_header.py +++ b/easybuild/scripts/add_header.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC-UGent team. diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index afa4e44e32..00762288f9 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/fix_broken_easyconfigs.py b/easybuild/scripts/fix_broken_easyconfigs.py index 9b613aa46b..42445bdff2 100755 --- a/easybuild/scripts/fix_broken_easyconfigs.py +++ b/easybuild/scripts/fix_broken_easyconfigs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 5105f6356f..ecf0313ad8 100644 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2016-2017 Ghent University +# Copyright 2016-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/generate_software_list.py b/easybuild/scripts/generate_software_list.py index c62d08abb1..7c5638f4e6 100644 --- a/easybuild/scripts/generate_software_list.py +++ b/easybuild/scripts/generate_software_list.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 337e19dfff..e8d50fa1fb 100644 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/port_easyblock.py b/easybuild/scripts/port_easyblock.py index aadc6c270a..9eed030757 100644 --- a/easybuild/scripts/port_easyblock.py +++ b/easybuild/scripts/port_easyblock.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/prep_for_release.py b/easybuild/scripts/prep_for_release.py index 11f586a428..6d22ee4ef9 100644 --- a/easybuild/scripts/prep_for_release.py +++ b/easybuild/scripts/prep_for_release.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/repo_setup.py b/easybuild/scripts/repo_setup.py index 4196febd37..f5d7d5e56e 100644 --- a/easybuild/scripts/repo_setup.py +++ b/easybuild/scripts/repo_setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 24136d4b4a..f63a3a698b 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2017 Ghent University +# Copyright 2016-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index 7151b6965c..3b5fef9138 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index 6c2751ecf7..4c2a2703d6 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index fa91d34417..efe1a03b92 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index 81151ea6c9..cb23ae88a5 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index fc35409c7b..41c90e8cbb 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 41149b2fe0..419e3d4d44 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index 5ae90daf45..24c6f8d488 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index 3bdef87c33..38e18d238d 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 49cb547c2a..abcebe9511 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index ab76c636d5..8e1be062c9 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index 1adab89133..4b36275cd1 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 2d597bd40a..0a00d09fea 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index 329f71cd2a..9dfc4cd8e3 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 613f67fbc4..f3d2dc19c1 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 7439e60d70..0ec3cde9f1 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py index 2f83c015c9..7541088a61 100644 --- a/easybuild/toolchains/craycce.py +++ b/easybuild/toolchains/craycce.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py index b653d71132..dba23c83d7 100644 --- a/easybuild/toolchains/craygnu.py +++ b/easybuild/toolchains/craygnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py index 1f7ff3580c..7c834ef976 100644 --- a/easybuild/toolchains/crayintel.py +++ b/easybuild/toolchains/crayintel.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py index 099eca7012..3d82e3c99a 100644 --- a/easybuild/toolchains/craypgi.py +++ b/easybuild/toolchains/craypgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2015 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index 8d28730fab..60a26bf667 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index f7bf293008..91b5fc7ce1 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index d5056191fd..07849b6d49 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 3f9738f387..f18e375ace 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index 2bca053b70..8896dd379a 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index bbe44e2e4a..26f07ea028 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index 19cbea6649..a61a152f3d 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index 20d7c1af28..99b010adf7 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index a9d848d62f..ead8f081e7 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index b0afc1f81f..61f90b996f 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py index 727f17f24f..d48050915d 100644 --- a/easybuild/toolchains/gimpic.py +++ b/easybuild/toolchains/gimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 8ba326c2a5..1350617128 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index d97de1ea05..654eebfe55 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index 8e92c93dc9..7b17b91a12 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 3432d45749..0bbaac4b61 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index c61032e51e..11b0d8acfe 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index fac3d0f47e..949a0f9b92 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index cadd2a71d7..6bde71abba 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 4c441d08ae..4ec094689a 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py index a35b5d27a2..40a4c9aff6 100644 --- a/easybuild/toolchains/gnu.py +++ b/easybuild/toolchains/gnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index 31d068b4c0..64caab77f6 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index 25236be2b2..9b78077e0f 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index f45a94c7dc..c5270ff10b 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 896e4938b6..7ddc1cc9c6 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index fb0bad1d6d..e54d4353a9 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index 998b20119f..bfe26e7744 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index 6fd8342db8..c53d79274a 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index e01545ffe4..98e7a85190 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index a135b74045..8449d22bb7 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 8d0cf42412..e6c0ea3372 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index 133a23948a..a9f9441f3b 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py index e78c6c1b19..6d942d1602 100644 --- a/easybuild/toolchains/iccifortcuda.py +++ b/easybuild/toolchains/iccifortcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index b46f686159..2aa6c32dd9 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 916c03e99e..0738421f16 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 403c43375a..aef1e57273 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index fde7175f6b..07a41ee514 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index c4c3b5e5bb..bb2074d5ae 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index 1f11884dfe..826a1a045c 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 50b237e79f..37e8c0815b 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index 1d2abcf50e..7888bce085 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index b8d25b212a..04fb90f47b 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index f20bbbbc18..8d42363920 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index 5563aaf932..1f7d1a42aa 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index e342bfd62e..e7e38d907f 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 3538a8278b..70f4ba842c 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index a35d262239..52c1cf0f06 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index 9b4da44137..e140e82b5b 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index 1626ed7e18..48d3ceaae6 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index d522beee55..437e724581 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index 6cbfba8574..53e0e8ef52 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index 8047a94c6c..1c2426d7a1 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 75ed5d782f..9e1a7e0887 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 7550b2b051..da1f9cb825 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 364b06ef0d..fba7f4f544 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index fb22151ec5..d89300adc0 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index 7e6109fe47..28066ca9f6 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 91af8256b3..0819991ff6 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index 937107ac9b..eb9d651a69 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index 883a02642b..d15cc2fa26 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 72920188c1..32df478fe0 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index a601a06449..bb0569c659 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index 96c7d5736c..6a6cc5b213 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index 76b8a5808c..cb714b5184 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index f6f0d14d76..c67d986d3b 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 7be4167cd0..a9434a1f57 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py index 94a329d571..fa247c8bae 100644 --- a/easybuild/toolchains/mpi/psmpi.py +++ b/easybuild/toolchains/mpi/psmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index bcfca5981f..33657639a8 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index ac717a6706..5e9bdb3df9 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index e1e673bc70..e71c17d1a3 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py index 64d0896cbb..bf8f079803 100644 --- a/easybuild/toolchains/pompi.py +++ b/easybuild/toolchains/pompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index 89c6b5ea4a..a6b04cbbb8 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index 105ff98ed2..45c1ce9329 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index ffb09d5466..f06dd1c2c7 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index dc0f45360d..8979eb2840 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 5f133b6604..f73da0351c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index e7dc86a907..6464325cef 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/deprecated/__init__.py b/easybuild/tools/deprecated/__init__.py index 505d2e7767..b10f75165f 100644 --- a/easybuild/tools/deprecated/__init__.py +++ b/easybuild/tools/deprecated/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 3498dd726f..741ad83ed4 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index c6d67bd406..6a3a889b5d 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index f80934f0ec..5176ec7180 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c1cd3b206d..3fd5b749e4 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 05019a647a..7c7b97b817 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2017 Ghent University +# Copyright 2018-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 2e8a818049..b7d15655ee 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index a512df87c5..6051c15299 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/__init__.py b/easybuild/tools/job/__init__.py index 3af278bf54..927692d06a 100644 --- a/easybuild/tools/job/__init__.py +++ b/easybuild/tools/job/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index bde520a394..cba23f772d 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index ec2b4e1e14..4eb897ed3d 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # Copyright 2015 S3IT, University of Zurich # # This file is part of EasyBuild, diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index 49bf7ddb53..14225825af 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c7766ab4da..e4715a4b51 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index 6346949299..71996a2e7a 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py index 52e8f39dcb..20fa513884 100644 --- a/easybuild/tools/module_naming_scheme/categorized_mns.py +++ b/easybuild/tools/module_naming_scheme/categorized_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2016 Ghent University +# Copyright 2016-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index 0f4813bbf5..17875aaf89 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 2d322c1f65..6b7ac61442 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py index 23ff693f51..e5d31cbcd8 100644 --- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 62c524353e..f67740f755 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 6409a43422..4d6e686cf7 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index edf928cce6..cdc9c91d93 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 451e2a9aa7..8f30c92df2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index 3e320b9f7e..d2e65c7439 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5cf66c93e8..85d0620314 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/__init__.py b/easybuild/tools/package/__init__.py index 973c4ba767..75f3de5422 100644 --- a/easybuild/tools/package/__init__.py +++ b/easybuild/tools/package/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/__init__.py b/easybuild/tools/package/package_naming_scheme/__init__.py index 0111e2581e..70db185021 100644 --- a/easybuild/tools/package/package_naming_scheme/__init__.py +++ b/easybuild/tools/package/package_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py index 66ca2199c2..934a8901c1 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py index 8e3b9cd6a1..bcbd6e8c36 100644 --- a/easybuild/tools/package/package_naming_scheme/pns.py +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 599bf442a7..f570f96e55 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index c9075e70a2..1a5018a3e8 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/__init__.py b/easybuild/tools/repository/__init__.py index a592af8109..bf941f9e40 100644 --- a/easybuild/tools/repository/__init__.py +++ b/easybuild/tools/repository/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 4751b846ff..0d287a4cb8 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 8b80f4b937..63c47ffe27 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 6330a6d587..3d27e5ae4b 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index 52154ba3b4..25aba7a430 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index db1d93ce01..228894b262 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index b82907a1a2..f797aa0a6e 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d0a94e6ffc..c8413470e5 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 471d0614a2..c82dabf563 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 162bab391b..62f7061372 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index da40317a18..6c46e27e78 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 8cbfdcc2f3..67bd0348d6 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 2d3a888f93..6a2dc9e209 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index de91ff99b7..385948a98a 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 599b76bfec..5d4f617dd5 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 22c6790dce..e384f214e0 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 08ac190d31..2350bd423d 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index eb4bc420e2..788cef1430 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index e6f5eb5d60..7b045f0cb7 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 61929c0166..7aef21377d 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 9b90f6e967..778a4ef6ae 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index afdbdcb50b..b91acfe3dd 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index 552b3eb739..0a9f514866 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index b244487272..22293c9b1c 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/setup.py b/setup.py index 1801ff92c2..03500d8ba9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/__init__.py b/test/__init__.py index 4ba4dfe401..366ec28047 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index dd77cc77f3..12c59136cf 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index 76dd1f1ad3..737de4a363 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 370e00a459..759d79b4d6 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/config.py b/test/framework/config.py index 264a19d68e..29a632acef 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/docs.py b/test/framework/docs.py index 4fd551b0eb..29c2cd590c 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index ca658b1cf0..f9af775402 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b07c743ebe..6feeacc38c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index ebf50841a3..ff96beb8f4 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index f11320828e..b8ad4cbfa2 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 91e0082cef..781491d843 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index 9c52e92020..e1ec9d9be1 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/environment.py b/test/framework/environment.py index 355af307cc..7fb6ad3e46 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8a13afb1ae..d894b63a0a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 935ab59e7a..32224551bc 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index ae344e4f9d..fc890310a2 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/github.py b/test/framework/github.py index dfc1a1dc8d..89fd2ba5f5 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 993bcfa00d..ca37dff26e 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2017 Ghent University +# Copyright 2018-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/include.py b/test/framework/include.py index a2389d726b..7e50df53f6 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index ba720cb449..17925fde9d 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 6871168f3d..d9b9bfe0d5 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modules.py b/test/framework/modules.py index 780c88aa85..d38dcef0b4 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 3a5feb9707..4c5adf0073 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2017 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/options.py b/test/framework/options.py index edf15fe62e..13e5ee09f6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/package.py b/test/framework/package.py index 3c59668bdc..5a6c3ac3f6 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 4b0b8d9d6a..78d43da20b 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2014 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/repository.py b/test/framework/repository.py index 5501e330a2..1b14a5de0c 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/robot.py b/test/framework/robot.py index e4f3b9b657..8512e6f709 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/run.py b/test/framework/run.py index e05f18d2a1..d49fb28a76 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py index b5e6050630..c330617a4a 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index dc156dd7e9..0f79da96f5 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py index e83adb0156..f408bb53e8 100644 --- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py +++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 6582c1e4b3..aa19dd6013 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 6d7e0ef6df..64c1a38d99 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index 7313701b22..2ee411743b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index ffae61e272..0def6c1b77 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index e1a1a438c0..6f344759df 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py index ce5da97db2..af59824943 100644 --- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py index 6194c5df06..c4a7e11577 100644 --- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 6d39b41e1b..718e61bda3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index aa53530bd8..8d2ee59030 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index e6ab78e23d..1ae310b11f 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2017 Ghent University +# Copyright 2009-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index 153a15c47d..7287f2692c 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2017 Ghent University +# Copyright 2011-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py index 531097ae0e..5b31c91082 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2014 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 728b94447a..7ee6638a46 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index ab1a3ed640..ac507a16d6 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 7f95ec856c..4a54cf91a0 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -1,5 +1,5 @@ # # -# Copyright 2014 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/style.py b/test/framework/style.py index 5535f5202f..aa4cbe64f1 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2017 Ghent University +# Copyright 2016-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/suite.py b/test/framework/suite.py index bd41a3ae57..6162dba967 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 091e7871a4..7c3c171ec3 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 3399a51dec..df524da33b 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index cd107aef51..6615903fd7 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index fa88ba06ce..6c30cea045 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2017 Ghent University +# Copyright 2013-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 85f1027da7..21f855d540 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2014 Ghent University +# Copyright 2014-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 78a827ba5a..7cd31d52b9 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 0c526b67a0..5eeac6b6fa 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/variables.py b/test/framework/variables.py index dda477dcac..3a1585517a 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 4d077b6bfb..a6b8f3ea3a 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2017 Ghent University +# Copyright 2015-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 867d8bb59c0c44f83a020b29c73abc0929bb0732 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jan 2018 20:24:22 +0100 Subject: [PATCH 096/256] fix remark --- easybuild/tools/hooks.py | 2 +- test/framework/hooks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 7c7b97b817..d379aeeb6e 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2018 Ghent University +# Copyright 2017-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index ca37dff26e..075020a526 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2018 Ghent University +# Copyright 2017-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 01548febdea86bbc0c76263fd88ad09f6acb4c5a Mon Sep 17 00:00:00 2001 From: Riccardo Murri Date: Fri, 12 Jan 2018 14:52:32 +0100 Subject: [PATCH 097/256] Make code in `tools/job/gc3pie.py` forward-compatible with GC3Pie 2.5 --- easybuild/tools/job/gc3pie.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 4eb897ed3d..0a8b286856 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -219,6 +219,15 @@ def complete(self): # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True + # `Engine.stats()` (which is used later on in `_print_status_report()`) + # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible + # with both + try: + self._engine.init_stats_for(Application) + except AttributeError: + _log.debug("No `init_stats_for` method in the Engine class;" + " assuming pre-2.5.0 GC3Pie and ignoring error.") + # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. From 53dcea1e709120dd3cdb1290fb39bfef4e5bcb7f Mon Sep 17 00:00:00 2001 From: John Donners Date: Fri, 12 Jan 2018 18:40:19 +0100 Subject: [PATCH 098/256] Added functionality to skip devel module with naming scheme --- easybuild/framework/easyblock.py | 4 ++++ easybuild/tools/module_naming_scheme/mns.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 806061fe84..2e0073b21d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -923,6 +923,10 @@ def make_devel_module(self, create_in_builddir=False): WARNING: you cannot unload using $EBDEVELNAME (for now: use module unload `basename $EBDEVELNAME`) """ + if not ActiveMNS().mns.det_generate_devel_module(): + self.log.info("Skipping devel module...") + return + self.log.info("Making devel module...") # load fake module diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index f67740f755..ef426879b8 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -164,3 +164,11 @@ def is_short_modname_for(self, short_modname, name): short_modname, name, modname_regex.pattern, res) return res + + def det_generate_devel_module(self): + """ + Determine if a devel module should be generated. Can be used to create a separate set of modules with a different naming scheme. + Software is already installed beforehand with one naming scheme, including development module. + """ + return True + From a98637bbe45df5993cc322abd035b70b78cc3f86 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 15 Jan 2018 17:05:57 +0800 Subject: [PATCH 099/256] prepare release notes for eb351 --- RELEASE_NOTES | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index f6993d89bc..6ec835c8c2 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,18 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v3.5.1 (January 16th 2018) +-------------------------- + +bugfix release +- various enhancements, including: + - add definition of giolfc toolchain (#2359) + - add support for Environment Modules 4 (#2365) +- various bug fixes, including: + - install SQLAlchemy < 1.2.0 with Python 2.6 in Travis config (#2367) + - make code in `tools/job/gc3pie.py` forward-compatible with GC3Pie 2.5 (#2373) + + v3.5.0 (December 15th 2017) --------------------------- From af13e05522c6e1b40106eb9a70a1a3ab186b1b2f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Jan 2018 21:09:34 +0100 Subject: [PATCH 100/256] bump version to 3.5.1 --- RELEASE_NOTES | 2 +- easybuild/tools/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 6ec835c8c2..72ee6f1a3d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -12,7 +12,7 @@ bugfix release - add support for Environment Modules 4 (#2365) - various bug fixes, including: - install SQLAlchemy < 1.2.0 with Python 2.6 in Travis config (#2367) - - make code in `tools/job/gc3pie.py` forward-compatible with GC3Pie 2.5 (#2373) + - make code in easybuild/tools/job/gc3pie.py forward-compatible with GC3Pie 2.5 (#2373) v3.5.0 (December 15th 2017) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 22293c9b1c..36490768f8 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.1.dev0') +VERSION = LooseVersion('3.5.1') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 0fc6ae0fd1cd15c521198181e5178d906d2ecd47 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Jan 2018 16:34:19 +0100 Subject: [PATCH 101/256] bump version to 3.5.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 36490768f8..bba6b6656a 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.1') +VERSION = LooseVersion('3.5.2.dev0') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 8966e300520863b89997704d11f7beb13936185f Mon Sep 17 00:00:00 2001 From: Riccardo Murri Date: Tue, 16 Jan 2018 17:01:58 +0100 Subject: [PATCH 102/256] New command-line option `--job-max-jobs` to cap nr of submitted build jobs. --- easybuild/tools/job/gc3pie.py | 3 +++ easybuild/tools/options.py | 1 + 2 files changed, 4 insertions(+) diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 0a8b286856..2c47c8b38f 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -219,6 +219,9 @@ def complete(self): # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True + # some sites may not be happy with flooding the cluster with build jobs... + self._engine.limit_in_flight = build_option('job_max_jobs') + # `Engine.stats()` (which is used later on in `_print_status_report()`) # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible # with both diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85d0620314..9e12020271 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -634,6 +634,7 @@ def job_options(self): opts = OrderedDict({ 'backend-config': ("Configuration file for job backend", None, 'store', None), 'cores': ("Number of cores to request per job", 'int', 'store', None), + 'max-jobs': ("Maximum number of concurrent jobs (queued or running)", 'int', 'store', 50), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), 'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0), From 07a50e75a0bdda86dd76087a0ad79f366fc9fb10 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Tue, 16 Jan 2018 21:27:16 +0100 Subject: [PATCH 103/256] Add pagination support in clean_gists.py --- easybuild/scripts/clean_gists.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index 3b94ffc2de..41926a5283 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -72,21 +72,28 @@ def main(): token = fetch_github_token(username) gh = RestClient(GITHUB_API_URL, username=username, token=token) - # ToDo: add support for pagination - status, gists = gh.gists.get(per_page=100) - - if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s", - username, status, gists) - else: - log.info("Found %s gists", len(gists)) + all_gists = [] + cur_page = 1 + while True: + status, gists = gh.gists.get(per_page=100, page=cur_page) + + if status != HTTP_STATUS_OK: + raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s", + username, status, gists) + if gists: + all_gists.extend(gists) + cur_page += 1 + else: + break + + log.info("Found %s gists", len(all_gists)) regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") pr_cache = {} num_deleted = 0 - for gist in gists: + for gist in all_gists: if not gist["description"]: continue re_pr_num = regex.search(gist["description"]) From 1de5904ad41613891424ab69aae98cd7abfd7ce7 Mon Sep 17 00:00:00 2001 From: Riccardo Murri Date: Wed, 17 Jan 2018 16:17:48 +0100 Subject: [PATCH 104/256] Need to add `jobs_max_jobs` to `BUILD_OPTIONS_CMDLINE` as well. --- easybuild/tools/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f73da0351c..1d6b657eb7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -127,6 +127,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'ignore_dirs', 'job_backend_config', 'job_cores', + 'job_max_jobs', 'job_max_walltime', 'job_output_dir', 'job_polling_interval', From 81dd24cf87a125f9359980ab63638810146ac720 Mon Sep 17 00:00:00 2001 From: Riccardo Murri Date: Thu, 18 Jan 2018 16:40:30 +0100 Subject: [PATCH 105/256] The correct parameter for capping nr. of concurrent jobs is `gc3libs.Engine.max_in_flight` Many thanks to Theofilos Manitaras for noticing and suggesting the fix! --- easybuild/tools/job/gc3pie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 2c47c8b38f..11f1ff40e7 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -220,7 +220,7 @@ def complete(self): self._engine.retrieve_overwrites = True # some sites may not be happy with flooding the cluster with build jobs... - self._engine.limit_in_flight = build_option('job_max_jobs') + self._engine.max_in_flight = build_option('job_max_jobs') # `Engine.stats()` (which is used later on in `_print_status_report()`) # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible From ab9497a81431b378b69a6b5827eefb2da5ecd69c Mon Sep 17 00:00:00 2001 From: John Donners Date: Mon, 22 Jan 2018 11:05:41 +0100 Subject: [PATCH 106/256] Pushed check det_make_devel_module up to make_module_step --- easybuild/framework/easyblock.py | 9 ++++----- easybuild/tools/module_naming_scheme/mns.py | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 2e0073b21d..68eb4cc91e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -923,10 +923,6 @@ def make_devel_module(self, create_in_builddir=False): WARNING: you cannot unload using $EBDEVELNAME (for now: use module unload `basename $EBDEVELNAME`) """ - if not ActiveMNS().mns.det_generate_devel_module(): - self.log.info("Skipping devel module...") - return - self.log.info("Making devel module...") # load fake module @@ -2328,8 +2324,11 @@ def make_module_step(self, fake=False): mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) - if not fake: + if ActiveMNS().mns.det_make_devel_module() and not fake: self.make_devel_module() + else: + self.log.info("Skipping devel module...") + if build_option('set_default_module'): self._set_module_as_default() diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index ef426879b8..6d21bb3df0 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -165,9 +165,10 @@ def is_short_modname_for(self, short_modname, name): return res - def det_generate_devel_module(self): + def det_make_devel_module(self): """ - Determine if a devel module should be generated. Can be used to create a separate set of modules with a different naming scheme. + Determine if a devel module should be generated. + Can be used to create a separate set of modules with a different naming scheme. Software is already installed beforehand with one naming scheme, including development module. """ return True From a6cb99ed98a594f76f7b52d65f5913a14ab72933 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 31 Jan 2018 12:42:30 +0000 Subject: [PATCH 107/256] Allow basic compiler modulenames to be specified as keys in optarch. Examples: 'GCCcore', 'icc', 'ifort'. They will be considered before the compiler family ('Intel', etc). --- easybuild/tools/toolchain/compiler.py | 15 +++++++++------ test/framework/toolchain.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 67bd0348d6..0ce8d695e3 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -291,14 +291,17 @@ def _set_optimal_architecture(self, default_optarch=None): # optarch has been validated as complex string with multiple compilers and converted to a dictionary elif isinstance(optarch, dict): - current_compiler = getattr(self, 'COMPILER_FAMILY', None) - if current_compiler in optarch: - if optarch[current_compiler] == OPTARCH_GENERIC: - use_generic = True - else: + # first try module names, than the family in optarch + current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + + [getattr(self, 'COMPILER_FAMILY', None)]) + for current_compiler in current_compiler_names: + if current_compiler in optarch: optarch = optarch[current_compiler] + break + if optarch == OPTARCH_GENERIC: + use_generic = True # no option for this compiler - else: + if isinstance(optarch, dict): optarch = None self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", current_compiler) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index df524da33b..5a7add78a9 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -387,16 +387,18 @@ def test_compiler_dependent_optarch(self): """Test whether specifying optarch on a per compiler basis works.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')] - gcc_options = [('gccflag', 'gccflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] - toolchains = [('iccifort', '2011.13.367'), ('GCC', '4.7.2'), ('PGI', '16.7-GCC-5.4.0-2.26')] + gcc_options = [('gccflag', 'gccflag'), ('-ftree-vectorize', '-ftree-vectorize'), ('', '')] + gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] + toolchains = [('iccifort', '2011.13.367'), ('GCC', '4.7.2'), ('GCCcore', '6.2.0'), ('PGI', '16.7-GCC-5.4.0-2.26')] enabled = [True, False] - test_cases = product(intel_options, gcc_options, toolchains, enabled) + test_cases = product(intel_options, gcc_options, gcccore_options, toolchains, enabled) - for (intel_flags, intel_flags_exp), (gcc_flags, gcc_flags_exp), (toolchain, toolchain_ver), enable in test_cases: + for (intel_flags, intel_flags_exp), (gcc_flags, gcc_flags_exp), (gcccore_flags, gcccore_flags_exp), (toolchain, toolchain_ver), enable in test_cases: optarch_var = {} optarch_var['Intel'] = intel_flags optarch_var['GCC'] = gcc_flags + optarch_var['GCCcore'] = gcccore_flags build_options = {'optarch': optarch_var} init_config(build_options=build_options) tc = self.get_toolchain(toolchain, version=toolchain_ver) @@ -407,6 +409,8 @@ def test_compiler_dependent_optarch(self): flags = intel_flags_exp elif toolchain == 'GCC': flags = gcc_flags_exp + elif toolchain == 'GCCcore': + flags = gcccore_flags_exp else: # PGI as an example of compiler not set # default optarch flag, should be the same as the one in # tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch,tc.cpu_family)] @@ -423,6 +427,8 @@ def test_compiler_dependent_optarch(self): intel_options[1][1], gcc_options[0][1], gcc_options[1][1], + gcccore_options[0][1], + gcccore_options[1][1], 'xHost', # default optimal for Intel 'march=native', # default optimal for GCC ] From a453f70b0db45a8151ed25a81d21ac509212755e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 07:21:27 +0000 Subject: [PATCH 108/256] Fix and clarify comments. --- easybuild/tools/toolchain/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 0ce8d695e3..3dcf83d5e1 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -291,7 +291,7 @@ def _set_optimal_architecture(self, default_optarch=None): # optarch has been validated as complex string with multiple compilers and converted to a dictionary elif isinstance(optarch, dict): - # first try module names, than the family in optarch + # first try module names, then the family in optarch current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + [getattr(self, 'COMPILER_FAMILY', None)]) for current_compiler in current_compiler_names: @@ -300,7 +300,7 @@ def _set_optimal_architecture(self, default_optarch=None): break if optarch == OPTARCH_GENERIC: use_generic = True - # no option for this compiler + # still a dict: no option for this compiler if isinstance(optarch, dict): optarch = None self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", From e530fb755e00a70e9ebc59f5d976941dbb8f0f0e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 07:28:02 +0000 Subject: [PATCH 109/256] Shuffle code a little to only test once for GENERIC optarch. --- easybuild/tools/toolchain/compiler.py | 34 +++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 3dcf83d5e1..3147e9ba86 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -280,31 +280,29 @@ def _set_optimal_architecture(self, default_optarch=None): :param default_optarch: default value to use for optarch, rather than using default value based on architecture (--optarch and --optarch=GENERIC still override this value) """ - use_generic = False optarch = build_option('optarch') # --optarch is specified with flags to use + if optarch is not None and isinstance(optarch, dict): + # optarch has been validated as complex string with multiple compilers and converted to a dictionary + # first try module names, then the family in optarch + current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + + [getattr(self, 'COMPILER_FAMILY', None)]) + for current_compiler in current_compiler_names: + if current_compiler in optarch: + optarch = optarch[current_compiler] + break + # still a dict: no option for this compiler + if isinstance(optarch, dict): + optarch = None + self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", + current_compiler) + + use_generic = False if optarch is not None: # optarch has been parsed as a simple string if isinstance(optarch, basestring): if optarch == OPTARCH_GENERIC: use_generic = True - - # optarch has been validated as complex string with multiple compilers and converted to a dictionary - elif isinstance(optarch, dict): - # first try module names, then the family in optarch - current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + - [getattr(self, 'COMPILER_FAMILY', None)]) - for current_compiler in current_compiler_names: - if current_compiler in optarch: - optarch = optarch[current_compiler] - break - if optarch == OPTARCH_GENERIC: - use_generic = True - # still a dict: no option for this compiler - if isinstance(optarch, dict): - optarch = None - self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", - current_compiler) else: raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch) From 2c02ab40a65c0996d24bcfc5528419369a28ed48 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jan 2018 17:36:43 +0100 Subject: [PATCH 110/256] update bootstrap script to be compatible with Modules v4 --- .travis.yml | 10 +++------- easybuild/scripts/bootstrap_eb.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 804ecab70a..27e2de2ac7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,15 +93,11 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20170808.01 7a22faf5ee9d249807fd883e435008fb84c11129425ec09d3cc76918b36ef3d5" + - EB_BOOTSTRAP_EXPECTED="20180105.01 ac138fa34ecf488f28955fa174038a141fdd3cc74453849944a78d1704a5bc88" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - - if [ "x$TEST_EASYBUILD_MODULES_TOOL" != 'xEnvironmentModules' ]; then - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; - fi; + - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH # simply sanity check on bootstrapped EasyBuild module - - if [ "x$TEST_EASYBUILD_MODULES_TOOL" != 'xEnvironmentModules' ]; then - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version; - fi; + - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 00762288f9..2a28484454 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -53,7 +53,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20170808.01' +EB_BOOTSTRAP_VERSION = '20180105.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -240,7 +240,7 @@ def check_module_command(tmpdir): def check_cmd_help(modcmd): """Check 'help' output for specified command.""" - modcmd_re = re.compile(r'module\s.*command\s') + modcmd_re = re.compile(r'module\s.*command') cmd = "%s python help" % modcmd os.system("%s > %s 2>&1" % (cmd, out)) txt = open(out, 'r').read() @@ -249,6 +249,7 @@ def check_cmd_help(modcmd): # order matters, which is why we don't use a dict known_module_commands = [ + ('modulecmd.tcl', 'EnvironmentModules'), ('lmod', 'Lmod'), ('modulecmd', 'EnvironmentModulesC'), ('modulecmd.tcl', 'EnvironmentModulesTcl'), @@ -267,6 +268,14 @@ def check_cmd_help(modcmd): easybuild_modules_tool = modtool info("Found module command '%s' via $LMOD_CMD (%s), so using it." % (modcmd, modtool)) break + elif modtool == 'EnvironmentModules': + # check value of $MODULESHOME as fallback + moduleshome = os.environ.get('MODULESHOME', 'MODULESHOME_NOT_DEFINED') + modcmd = os.path.join(moduleshome, 'libexec', 'modulecmd.tcl') + if os.path.exists(modcmd) and check_cmd_help(modcmd): + easybuild_modules_tool = modtool + info("Found module command '%s' via $MODULESHOME (%s), so using it." % (modcmd, modtool)) + break if easybuild_modules_tool is None: mod_cmds = [m for (m, _) in known_module_commands] From c220db17090b5a96c8ed95a8bab69fba7bc8ae1f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Jan 2018 14:20:19 +0100 Subject: [PATCH 111/256] also enable debugging of last part of stage2 if $EASYBUILD_BOOTSTRAP_DEBUG is defined --- easybuild/scripts/bootstrap_eb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 2a28484454..b8f44da11e 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -705,6 +705,9 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): from easybuild.main import main as easybuild_main easybuild_main() + if print_debug: + os.environ['EASYBUILD_DEBUG'] = '1' + # make sure the EasyBuild module was actually installed # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap, # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'... From f7a1d38169783821d7dab5f99100b3fb221b3aef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Jan 2018 19:44:45 +0100 Subject: [PATCH 112/256] bump version of bootstrap script to 20180117.01 --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27e2de2ac7..bd490e1532 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180105.01 ac138fa34ecf488f28955fa174038a141fdd3cc74453849944a78d1704a5bc88" + - EB_BOOTSTRAP_EXPECTED="20180117.01 ddb59fdc97e724c626a85001bd7a9a955b0c7b007f2e1d63a0750f0f7e59e12a" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index b8f44da11e..b29ee36cb2 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -53,7 +53,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180105.01' +EB_BOOTSTRAP_VERSION = '20180117.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False From b9d54521eb49c06de3cc105e8fb20e9634710592 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Jan 2018 21:06:03 +0100 Subject: [PATCH 113/256] check whether $MODULESHOME is defined to discriminate between EnvironmentModules and EnvironmentModulesTcl modules tool --- easybuild/scripts/bootstrap_eb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index b29ee36cb2..6868bf1c4e 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -249,16 +249,18 @@ def check_cmd_help(modcmd): # order matters, which is why we don't use a dict known_module_commands = [ - ('modulecmd.tcl', 'EnvironmentModules'), ('lmod', 'Lmod'), ('modulecmd', 'EnvironmentModulesC'), - ('modulecmd.tcl', 'EnvironmentModulesTcl'), + ('modulecmd.tcl', 'EnvironmentModules'), ] out = os.path.join(tmpdir, 'module_command.out') modtool = None for modcmd, modtool in known_module_commands: if check_cmd_help(modcmd): - easybuild_modules_tool = modtool + if 'MODULESHOME' in os.environ: + easybuild_modules_tool = 'EnvironmentModules' + else: + easybuild_modules_tool = 'EnvironmentModulesTcl' info("Found module command '%s' (%s), so using it." % (modcmd, modtool)) break elif modcmd == 'lmod': From 2653ff837b2f9009745613e3dfc152a17e76abdd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2018 15:11:26 +0100 Subject: [PATCH 114/256] fix checksum for bootstrap script in Travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bd490e1532..0c5de6300c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180117.01 ddb59fdc97e724c626a85001bd7a9a955b0c7b007f2e1d63a0750f0f7e59e12a" + - EB_BOOTSTRAP_EXPECTED="20180117.01 602704b07d8f9e75907ebf2823e9d70cef85c515985eddfbf98a88669cf86cf2" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap From cd29ab69755a79d28ad551f5d7aba93314f99201 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Thu, 1 Feb 2018 13:57:31 +0100 Subject: [PATCH 115/256] distinguish between EnvironmentModules and EnvironmentModulesTcl if command is modulecmd.tcl introduce a is_modulecmd_tcl_modulestcl def to make distinction between old Modules-Tcl and Modules 4+. --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0c5de6300c..cae004fdd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180117.01 602704b07d8f9e75907ebf2823e9d70cef85c515985eddfbf98a88669cf86cf2" + - EB_BOOTSTRAP_EXPECTED="20180117.01 c3e828769fe30f9e4798972f7ab052632047e5df3224e002d3d2599033678745" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 6868bf1c4e..5ac6dbeee4 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -247,6 +247,15 @@ def check_cmd_help(modcmd): debug("Output from %s: %s" % (cmd, txt)) return modcmd_re.search(txt) + def is_modulecmd_tcl_modulestcl(): + """Determine if modulecmd.tcl is EnvironmentModulesTcl.""" + modcmd_re = re.compile('Modules Release Tcl') + cmd = "modulecmd.tcl python --version" + os.system("%s > %s 2>&1" % (cmd, out)) + txt = open(out, 'r').read() + debug("Output from %s: %s" % (cmd, txt)) + return modcmd_re.search(txt) + # order matters, which is why we don't use a dict known_module_commands = [ ('lmod', 'Lmod'), @@ -257,10 +266,10 @@ def check_cmd_help(modcmd): modtool = None for modcmd, modtool in known_module_commands: if check_cmd_help(modcmd): - if 'MODULESHOME' in os.environ: - easybuild_modules_tool = 'EnvironmentModules' - else: - easybuild_modules_tool = 'EnvironmentModulesTcl' + # distinguish between EnvironmentModulesTcl and EnvironmentModules + if modcmd == 'modulecmd.tcl' and is_modulecmd_tcl_modulestcl(): + modtool = 'EnvironmentModulesTcl' + easybuild_modules_tool = modtool info("Found module command '%s' (%s), so using it." % (modcmd, modtool)) break elif modcmd == 'lmod': From 13c2fb681a0cb78bf0ccd82826d31fa6222ede45 Mon Sep 17 00:00:00 2001 From: Reese Baird Date: Thu, 1 Feb 2018 14:21:44 -0800 Subject: [PATCH 116/256] Initial set of OHPC module meta data for EasyBuild Signed-off-by: Reese Baird --- etc/ohpc_external_modules_metadata.cfg | 225 +++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 etc/ohpc_external_modules_metadata.cfg diff --git a/etc/ohpc_external_modules_metadata.cfg b/etc/ohpc_external_modules_metadata.cfg new file mode 100644 index 0000000000..9c6be9eab8 --- /dev/null +++ b/etc/ohpc_external_modules_metadata.cfg @@ -0,0 +1,225 @@ +# metadata useful to EasyBuild (easybuilders.github.io/easybuild) for modules provided by OpenHPC (github.com/openhpc/ohpc) + +[R/3.4.2] +name = R +version = 3.4.2 +prefix = R_DIR + +[adios/1.12.0] +name = adios +version = 1.12.0 +prefix = ADIOS_DIR + +[autotools] +name = Autoconf, Automake, Libtool +version = 2.69, 1.15, 2.4.6 +prefix = /opt/ohpc/pub/utils/autotools/bin + +[boost/1.65.1] +name = Boost +version = 1.65.1 +prefix = BOOST_DIR + +[clustershell/1.8] +name = ClusterShell +version = 1.8 +prefix = clustershell_DIR + +[cmake/3.9.2] +name = CMake +version = 3.9.2 +prefix = /opt/ohpc/pub/utils/cmake/3.9.2 + +[fftw/3.3.6] +name = FFTW +version = 3.3.6 +prefix = FFTW_DIR + +[gnu7/7.2.0] +name = GCC +version = 7.2.0 +prefix = /opt/ohpc/pub/compiler/gcc/7.2.0 + +[gsl/2.4] +name = gsl +version = 2.4 +prefix = GSL_DIR + +[hdf5/1.10.1] +name = HDF5 +version = 1.10.1 +prefix = HDF5_DIR + +[hwloc/1.11.8] +name = hwloc +version = 1.11.8 +prefix = HWLOC_DIR + +[hypre/2.11.2] +name = Hypre +version = 2.11.2 +prefix = HYPRE_DIR + +[impi/2018.1.163] +name = impi +version = 2018.1.163 +prefix = I_MPI_ROOT + +[intel/18.0.1.163] +name = intel +version = 2018 + +[llvm5/5.0.0] +name = LLVM +version = 5.0.0 +prefix = /opt/ohpc/pub/compiler/llvm/5.0.0 + +[metis/5.1.0] +name = METIS +version = 5.1.0 +prefix = METIS_DIR + +[mpi4py/2.0.0] +name = mpi4py +version = 2.0.0 +prefix = MPI4PY_DIR + +[mpiP/3.4.1] +name = mpiP +version = 3.4.1 +prefix = MPIP_DIR + +[mpich/3.2] +name = MPICH +version = 3.2 +prefix = MPI_DIR + +[mumps/5.1.1] +name = MUMPS +version = 5.1.1 +prefix = MUMPS_DIR + +[mvapich2/2.2] +name = MVAPICH2 +version = 2.2 +prefix = MPI_DIR + +[netcdf-cxx/4.3.0] +name = netCDF-C++ +version = 4.3.0 +prefix = NETCDF_CXX_DIR + +[netcdf-fortran/4.4.4] +name = netCDF-Fortran +version = 4.4.4 +prefix = NETCDF_FORTRAN_DIR + +[netcdf/4.4.1.1] +name = netCDF +version = 4.4.1.1 +prefix = NETCDF_DIR + +[numpy/1.13.1] +name = numpy +version = 1.13.1 +prefix = NUMPY_DIR + +[openblas/0.2.20] +name = OpenBLAS +version = 0.2.20 +prefix = OPENBLAS_DIR + +[openmpi3/3.0.0] +name = OpenMPI +version = 3.0.0 +prefix = MPI_DIR + +[papi/5.5.1] +name = PAPI +version = 5.5.1 +prefix = PAPI_DIR + +[pdtoolkit/3.24] +name = PDT +version = 3.24 +prefix = PDTOOLKIT_DIR + +[petsc/3.7.6] +name = PETSc +version = 3.7.6 +prefix = PETSC_DIR + +[plasma/2.8.0] +name = Plasma +version = 2.8.0 +prefix = PLASMA_DIR + +[pmix/1.2.3] +name = PMIx +version = 1.2.3 +prefix = PMIX_DIR + +[pnetcdf/1.8.1] +name = PnetCDF +version = 1.8.1 +prefix = PNETCDF_DIR + +[scalapack/2.0.2] +name = ScaLAPACK +version = 2.0.2 +prefix = SCALAPACK_DIR + +[scalasca/2.3.1] +name = Scalasca +version = 2.3.1 +prefix = SCALASCA_DIR + +[scipy/0.19.1] +name = scipy +version = 0.19.1 +prefix = SCIPY_DIR + +[scorep/3.1] +name = Score-P +version = 3.1 +prefix = SCOREP_DIR + +[scotch/6.0.4] +name = SCOTCH +version = 6.0.4 +prefix = SCOTCH_DIR + +[sionlib/1.7.1] +name = SIONlib +version = 1.7.1 +prefix = SIONLIB_DIR + +[slepc/3.7.4] +name = SLEPc +version = 3.7.4 +prefix = SLEPC_DIR + +[superlu/5.2.1] +name = SuperLU +version = 5.2.1 +prefix = SUPERLU_DIR + +[superlu_dist/4.2] +name = SuperLU_dist +version = 4.2 +prefix = SUPERLU_DIST_DIR + +[tau/2.26.1] +name = TAU +version = 2.26.1 +prefix = TAU_DIR + +[trilinos/12.10.1] +name = Trilinos +version = 12.10.1 +prefix = TRILINOS_DIR + +[valgrind/3.13.0] +name = Valgrind +version = 3.13.0 +prefix = VALGRIND_DIR From 7ffb93235e10ded26df92baf667a3b487a3540d4 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Fri, 2 Feb 2018 06:05:51 +0100 Subject: [PATCH 117/256] bump version of bootstrap script to 20180201.01 --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cae004fdd5..20b4f1ce77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180117.01 c3e828769fe30f9e4798972f7ab052632047e5df3224e002d3d2599033678745" + - EB_BOOTSTRAP_EXPECTED="20180201.01 9d6278409b3d9ecf034bd28384da25c14ebfe773606d4fa800e51ed11667e851" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 5ac6dbeee4..339c18c71a 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -53,7 +53,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180117.01' +EB_BOOTSTRAP_VERSION = '20180201.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False From 18d73dc1d5804032262255dd244ba80e588d41e5 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 31 Jan 2018 14:01:12 +0000 Subject: [PATCH 118/256] Add -ftree-vectorize to defaultopt='-O2' for GCC. Since Intel auto-vectorizes at -O2 but not at -O1, and GCC auto-vectorizes at -O3 but not at -O2 this makes it consistent. --- easybuild/toolchains/compiler/gcc.py | 1 + test/framework/options.py | 2 +- test/framework/toolchain.py | 10 +++++----- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index f3d2dc19c1..a494f96401 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -65,6 +65,7 @@ class Gcc(Compiler): 'defaultprec':[], 'loose': ['mrecip', 'mno-ieee-fp'], 'veryloose': ['mrecip=all', 'mno-ieee-fp'], + 'defaultopt': ['O2', 'ftree-vectorize'], } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/test/framework/options.py b/test/framework/options.py index 13e5ee09f6..1751fddff3 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2961,7 +2961,7 @@ def test_dump_env_config(self): "module load hwloc/1.6.2-GCC-4.7.2", # loading of dependency module # defining build env "export FC='gfortran'", - "export CFLAGS='-O2 -march=native'", + "export CFLAGS='-O2 -ftree-vectorize -march=native'", ] for pattern in patterns: regex = re.compile("^%s$" % pattern, re.M) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 5a7add78a9..44146250b4 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -481,7 +481,7 @@ def test_precision_flags(self): tc.set_options({}) tc.prepare() for var in flag_vars: - self.assertEqual(os.getenv(var), "-O2 -march=native") + self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native") # check other precision flags prec_flags = { @@ -498,9 +498,9 @@ def test_precision_flags(self): tc.prepare() for var in flag_vars: if enable: - self.assertEqual(os.getenv(var), "-O2 -march=native %s" % prec_flags[prec]) + self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native %s" % prec_flags[prec]) else: - self.assertEqual(os.getenv(var), "-O2 -march=native") + self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native") self.modtool.purge() def test_cgoolf_toolchain(self): @@ -580,7 +580,7 @@ def test_goolfc(self): tc.prepare() nvcc_flags = r' '.join([ - r'-Xcompiler="-O2 -%s -fopenmp"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)], + r'-Xcompiler="-O2 -ftree-vectorize -%s -fopenmp"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)], # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), @@ -905,7 +905,7 @@ def test_independence(self): 'CrayCCE': "-O2 -homp -craype-verbose", 'CrayGNU': "-O2 -fopenmp -craype-verbose", 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose", - 'GCC': "-O2 -test -fopenmp", + 'GCC': "-O2 -ftree-vectorize -test -fopenmp", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp", } From 1c8842b0e01d9faf91ab0637a26fbf567a1bcaaf Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 09:22:50 +0000 Subject: [PATCH 119/256] test_misc_flags_shared: split flags. This avoids seeing "-v" being used in "-ftree-vectorize". --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 44146250b4..97d294c249 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -277,7 +277,7 @@ def test_misc_flags_shared(self): # we need to make sure we check for flags, not letter (e.g. 'v' vs '-v') flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP[opt] for var in flag_vars: - flags = tc.get_variable(var) + flags = tc.get_variable(var).split() if enable: self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: From 2d44ee4e52e6c0b53c94693652786f6e3c2a14d6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 10:57:49 +0000 Subject: [PATCH 120/256] Use march=nocona instead of -ftree-vectorize in optarch test. As -ftree-vectorize is there by default it confuses the test case. --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 97d294c249..82d7d11954 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -387,7 +387,7 @@ def test_compiler_dependent_optarch(self): """Test whether specifying optarch on a per compiler basis works.""" flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')] - gcc_options = [('gccflag', 'gccflag'), ('-ftree-vectorize', '-ftree-vectorize'), ('', '')] + gcc_options = [('gccflag', 'gccflag'), ('march=nocona', 'march=nocona'), ('', '')] gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] toolchains = [('iccifort', '2011.13.367'), ('GCC', '4.7.2'), ('GCCcore', '6.2.0'), ('PGI', '16.7-GCC-5.4.0-2.26')] enabled = [True, False] From 17b41d6049934f5c3d4fadc4260c18558f22cf35 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Feb 2018 10:41:46 +0100 Subject: [PATCH 121/256] avoid fatal error when determining glibc version on non-glibc Linux system (e.g. Alphine Linux) --- easybuild/tools/systemtools.py | 13 +++++++------ test/framework/systemtools.py | 8 ++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index c82dabf563..5f89f4ba11 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -629,6 +629,7 @@ def get_glibc_version(): """ Find the version of glibc used on this system """ + glibc_ver = UNKNOWN os_type = get_os_type() if os_type == LINUX: @@ -637,16 +638,16 @@ def get_glibc_version(): res = glibc_ver_regex.search(glibc_ver_str) if res is not None: - glibc_version = res.group(1) - _log.debug("Found glibc version %s" % glibc_version) - return glibc_version + glibc_ver = res.group(1) + _log.debug("Found glibc version %s" % glibc_ver) else: - raise EasyBuildError("Failed to determine glibc version from '%s' using pattern '%s'.", - glibc_ver_str, glibc_ver_regex.pattern) + _log.warning("Failed to determine glibc version from '%s' using pattern '%s'.", + glibc_ver_str, glibc_ver_regex.pattern) else: # no glibc on OS X standard _log.debug("No glibc on a non-Linux system, so can't determine version.") - return UNKNOWN + + return glibc_ver def get_system_info(): diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 7c3c171ec3..f7e18c497f 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -332,6 +332,7 @@ def setUp(self): self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd self.orig_platform_uname = st.platform.uname + self.orig_get_tool_version = st.get_tool_version def tearDown(self): """Cleanup after systemtools test.""" @@ -341,6 +342,7 @@ def tearDown(self): st.get_os_type = self.orig_get_os_type st.run_cmd = self.orig_run_cmd st.platform.uname = self.orig_platform_uname + self.get_tool_version = self.orig_get_tool_version super(SystemToolsTest, self).tearDown() def test_avail_core_count_native(self): @@ -698,6 +700,12 @@ def test_glibc_version_linux(self): st.run_cmd = mocked_run_cmd self.assertEqual(get_glibc_version(), '2.12') + def test_glibc_version_linux_musl_libc(self): + """Test getting glibc version (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.get_tool_version = lambda _: "musl libc (x86_64); Version 1.1.18; Dynamic Program Loader" + self.assertEqual(get_glibc_version(), UNKNOWN) + def test_glibc_version_darwin(self): """Test getting glibc version (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN From 451778dc157914175d3644e91661dbdb1b71bb3c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Feb 2018 11:13:14 +0100 Subject: [PATCH 122/256] fix restoring of original get_tool_version function in systemtools test suite --- test/framework/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index f7e18c497f..9b5cc2a86a 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -342,7 +342,7 @@ def tearDown(self): st.get_os_type = self.orig_get_os_type st.run_cmd = self.orig_run_cmd st.platform.uname = self.orig_platform_uname - self.get_tool_version = self.orig_get_tool_version + st.get_tool_version = self.orig_get_tool_version super(SystemToolsTest, self).tearDown() def test_avail_core_count_native(self): From bdd8b48991dd40151f48cd9e84a48502ae2db056 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 12 Feb 2018 14:17:36 +0800 Subject: [PATCH 123/256] allow different target account in post_comment_in_issue --- easybuild/tools/github.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3fd5b749e4..c4c035d8e5 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -473,7 +473,7 @@ def create_gist(txt, fn, descr=None, github_user=None): return data['html_url'] -def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user=None): +def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCONFIGS_REPO, github_user=None): """Post a comment in the specified PR.""" if not isinstance(issue, int): try: @@ -492,7 +492,7 @@ def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user= github_token = fetch_github_token(github_user) g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) - pr_url = g.repos[GITHUB_EB_MAIN][repo].issues[issue] + pr_url = g.repos[account][repo].issues[issue] status, data = pr_url.comments.post(body={'body': txt}) if not status == HTTP_STATUS_CREATED: @@ -1013,7 +1013,7 @@ def merge_pr(pr): print_msg("\nReview %s merging pull request!\n" % ("OK,", "FAILed, yet forcibly")[force], prefix=False) comment = "Going in, thanks @%s!" % pr_data['user']['login'] - post_comment_in_issue(pr, comment, repo=pr_target_repo, github_user=github_user) + post_comment_in_issue(pr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) if dry_run: print_msg("[DRY RUN] Merged %s/%s pull request #%s" % (pr_target_account, pr_target_repo, pr), prefix=False) From 69ba9411608a6126b7e11fd15670301232c550b9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Feb 2018 14:48:27 +0100 Subject: [PATCH 124/256] declare support for RPATH linking stable --- easybuild/tools/toolchain/toolchain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 788cef1430..f58c229266 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -771,8 +771,6 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ - self.log.experimental("Using wrapper scripts for compiler/linker commands that enforce RPATH linking") - if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: From 31a6770e64a9258be0590b56e9e319f9442d2143 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Feb 2018 17:07:56 +0100 Subject: [PATCH 125/256] exclude location of RPATH wrapper script from $PATH to avoid fork bomb --- easybuild/scripts/rpath_wrapper_template.sh.in | 3 +++ easybuild/tools/toolchain/toolchain.py | 1 + 2 files changed, 4 insertions(+) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 0d0bd08903..573ac2a9d0 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -55,6 +55,9 @@ $rpath_args_out" # define $CMD_ARGS by evaluating output of rpath_args.py script eval $rpath_args_out +# exclude location of this wrapper from $PATH to avoid other potential wrappers calling this wrapper +export PATH=$(echo $PATH | tr ':' '\n' | grep -v "^%(wrapper_dir)s$" | tr '\n' ':') + # call original command with modified list of command line arguments log "running '%(orig_cmd)s $(echo ${CMD_ARGS[@]})'" %(orig_cmd)s "${CMD_ARGS[@]}" diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 788cef1430..47afcaefe1 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -831,6 +831,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None 'rpath_filter': rpath_filter, 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, + 'wrapper_dir': wrapper_dir, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) From c3c0aa0052b682723b1bf15d1eb81f84e7fcdeae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Feb 2018 17:43:09 +0100 Subject: [PATCH 126/256] install each RPATH wrapper in its own subdir --- easybuild/tools/toolchain/toolchain.py | 15 ++++++++++----- test/framework/toy_build.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 47afcaefe1..bd7624b6c5 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -761,7 +761,7 @@ def is_rpath_wrapper(path): """ Check whether command at specified location already is an RPATH wrapper script rather than the actual command """ - in_rpath_wrappers_dir = os.path.basename(os.path.dirname(path)) == RPATH_WRAPPERS_SUBDIR + in_rpath_wrappers_dir = os.path.basename(os.path.dirname(os.path.dirname(path))) == RPATH_WRAPPERS_SUBDIR calls_rpath_args = 'rpath_args.py $CMD' in read_file(path) return in_rpath_wrappers_dir and calls_rpath_args @@ -778,7 +778,8 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None else: raise EasyBuildError("RPATH linking is currently only supported on Linux") - wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) + # directory where all wrappers will be placed + wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() @@ -786,9 +787,6 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') - # prepend location to wrappers to $PATH - setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) - # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: @@ -811,6 +809,10 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue + # determine location for this wrapper + # each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately + wrapper_dir = os.path.join(wrappers_dir, '%s_wrapper' % cmd) + cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... @@ -836,6 +838,9 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) + + # prepend location to this wrapper to $PATH + setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6c30cea045..6967e18b68 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1647,7 +1647,7 @@ def test_toy_rpath(self): def grab_gcc_rpath_wrapper_filter_arg(): """Helper function to grab filter argument from last RPATH wrapper for 'gcc'.""" rpath_wrappers_dir = glob.glob(os.path.join(os.getenv('TMPDIR'), '*', '*', 'rpath_wrappers'))[0] - gcc_rpath_wrapper_txt = read_file(os.path.join(rpath_wrappers_dir, 'gcc')) + gcc_rpath_wrapper_txt = read_file(glob.glob(os.path.join(rpath_wrappers_dir, '*', 'gcc'))[0]) rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '([^ ]*)'.*", re.M) res = rpath_args_regex.search(gcc_rpath_wrapper_txt) From fe21d13ac93cfd5bbb987421c2c8cd1fd2953247 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 28 Feb 2018 12:47:07 +0800 Subject: [PATCH 127/256] prepare release notes for eb352 --- RELEASE_NOTES | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 72ee6f1a3d..0a4e198eb5 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,23 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v3.5.2 (March 2nd 2018) +----------------------- + +bugfix release +- various enhancements, including: + - add functionality to skip devel module with naming scheme (#2374) + - add pagination support in clean_gists.py (#2379) + - allow basic compiler modulenames to be specified as keys in optarch. (#2387) + - initial set of OHPC module meta data for EasyBuild (#2392) + - allow different target account in post_comment_in_issue (#2399) + - declare support for RPATH linking stable (#2409) +- various bug fixes, including: + - update bootstrap script to be compatible with Modules v4 (#2390) + - avoid fatal error when determining glibc version on non-glibc Linux system (e.g. Alphine Linux) (#2398) + - exclude location of RPATH wrappers from $PATH to avoid fork bomb (#2410) + + v3.5.1 (January 16th 2018) -------------------------- From 34943633d11e948e4a58b1f0dd31c6ab1eaefe08 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Feb 2018 08:20:22 +0100 Subject: [PATCH 128/256] bump version to 3.5.2 --- RELEASE_NOTES | 2 +- easybuild/tools/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 0a4e198eb5..17149a65a0 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,7 +10,7 @@ bugfix release - various enhancements, including: - add functionality to skip devel module with naming scheme (#2374) - add pagination support in clean_gists.py (#2379) - - allow basic compiler modulenames to be specified as keys in optarch. (#2387) + - allow basic compiler modulenames to be specified as keys in optarch (#2387) - initial set of OHPC module meta data for EasyBuild (#2392) - allow different target account in post_comment_in_issue (#2399) - declare support for RPATH linking stable (#2409) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index bba6b6656a..99a8507f91 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.2.dev0') +VERSION = LooseVersion('3.5.2') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 6f21c8fda400bde00f96a991b6d86ae1df4e6c57 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 31 Jan 2018 14:56:41 +0000 Subject: [PATCH 129/256] Support Lmod 7.6+ depends_on via recursive-module-unload-depends-on. --- easybuild/framework/easyblock.py | 7 +++- easybuild/framework/easyconfig/default.py | 1 + easybuild/tools/config.py | 2 ++ easybuild/tools/module_generator.py | 9 +++++ easybuild/tools/options.py | 2 ++ test/framework/module_generator.py | 15 ++++++++ test/framework/options.py | 42 ++++++++++++++--------- 7 files changed, 60 insertions(+), 18 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 68eb4cc91e..9d67ca4ecb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -936,6 +936,8 @@ def make_devel_module(self, create_in_builddir=False): # capture all the EBDEVEL vars # these should be all the dependencies and we should load them recursive_unload = self.cfg['recursive_module_unload'] + if self.cfg['recursive_module_unload_depends_on']: + recursive_unload = 'depends_on' for key in os.environ: # legacy support if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): @@ -1041,12 +1043,15 @@ def make_module_dep(self, unload_info=None): self.log.debug("List of retained deps to load in generated module: %s", deps) # include load statements for retained dependencies + recursive_unload = self.cfg['recursive_module_unload'] + if self.cfg['recursive_module_unload_depends_on']: + recursive_unload = 'depends_on' loads = [] for dep in deps: unload_modules = [] if dep in unload_info: unload_modules.append(unload_info[dep]) - loads.append(self.module_generator.load_module(dep, recursive_unload=self.cfg['recursive_module_unload'], + loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload, unload_modules=unload_modules)) # Force unloading any other modules diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index d45c3c81fc..e27feea11c 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -172,6 +172,7 @@ 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], + 'recursive_module_unload_depends_on' : [False, 'Use depends_on (Lmod 7.6+) for dependencies in generated module', MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f73da0351c..cd9b99380c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -236,6 +236,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): False: [ 'dry_run', 'recursive_mod_unload', + 'recursive_mod_unload_depends_on', 'retain_all_deps', 'silent', 'try_to_generate', @@ -386,6 +387,7 @@ def init_build_options(build_options=None, cmdline_options=None): 'check_osdeps': not cmdline_options.ignore_osdeps, 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, 'recursive_mod_unload': cmdline_options.recursive_module_unload, + 'recursive_mod_unload_depends_on': cmdline_options.recursive_module_unload_depends_on, 'retain_all_deps': retain_all_deps, 'validate': not cmdline_options.force, 'valid_module_classes': module_classes(), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e4715a4b51..a0cc1bf0b8 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -797,6 +797,15 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body.extend([self.unload_module(m).strip() for m in unload_modules]) body.append(self.LOAD_TEMPLATE) + # Lmod 7.6+ supports depends_on which does this most nicely: + if (build_option('recursive_mod_unload_depends_on') or + recursive_unload == 'depends_on'): + lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_depends_on_min = '7.6' + if (lmod_version != 'NOT_FOUND' and + LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + return '\ndepends_on("%(mod_name)s")\n' % {'mod_name': mod_name} + if build_option('recursive_mod_unload') or recursive_unload: # wrapping the 'module load' with an 'is-loaded or mode == unload' # guard ensures recursive unloading while avoiding load storms, diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85d0620314..9428d70eee 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -474,6 +474,8 @@ def config_options(self): None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), + 'recursive-module-unload-depends-on': ("Use depends_on (Lmod 7.6+) for dependencies in all generated modules.", + None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index d9b9bfe0d5..b342d57d15 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -33,6 +33,7 @@ import sys import tempfile from unittest import TextTestRunner, TestSuite +from distutils.version import LooseVersion from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from vsc.utils.missing import nub @@ -283,6 +284,20 @@ def test_load(self): init_config(build_options={'recursive_mod_unload': True}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + # Lmod 7.6+ depends_on + lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_depends_on_min = '7.6' + if (lmod_version != 'NOT_FOUND' and + LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + expected = '\n'.join([ + '', + 'depends_on("mod_name")', + '', + ]) + self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on")) + init_config(build_options={'recursive_mod_unload_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name")) + def test_unload(self): """Test unload part in generated module file.""" diff --git a/test/framework/options.py b/test/framework/options.py index 13e5ee09f6..e057469434 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -35,6 +35,7 @@ import tempfile from unittest import TextTestRunner from urllib2 import URLError +from distutils.version import LooseVersion import easybuild.main import easybuild.tools.build_log @@ -1141,23 +1142,30 @@ def test_recursive_module_unload(self): eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') # check log message with --skip for existing module - args = [ - eb_file, - '--sourcepath=%s' % self.test_sourcepath, - '--buildpath=%s' % self.test_buildpath, - '--installpath=%s' % self.test_installpath, - '--debug', - '--force', - '--recursive-module-unload', - ] - self.eb_main(args, do_build=True, verbose=True) - - toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') - if get_module_syntax() == 'Lua': - toy_module += '.lua' - toy_module_txt = read_file(toy_module) - is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) - self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) + lastargs = ['--recursive-module-unload'] + lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_depends_on_min = '7.6' + if (lmod_version != 'NOT_FOUND' and + LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + lastargs.append(lastargs[0]+'-depends-on') + for lastarg in lastargs: + args = [ + eb_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--force', + lastarg, + ] + self.eb_main(args, do_build=True, verbose=True) + + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': + toy_module += '.lua' + toy_module_txt = read_file(toy_module) + is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) + self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) def test_tmpdir(self): """Test setting temporary directory to use by EasyBuild.""" From fcb0e21e3754c6cbcb5fa3f24b70d06070dfb48d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 15:12:32 +0000 Subject: [PATCH 130/256] Bump Lmod version used for Travis testing from 7.4 to 7.6. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20b4f1ce77..98a362d46a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ env: - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - LMOD_VERSION=6.6.3 - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - LMOD_VERSION=7.4 - - LMOD_VERSION=7.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - LMOD_VERSION=7.6 + - LMOD_VERSION=7.6 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules From 800bd90bc8c6d0b0b6a28840bab7f643778c08bb Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 15:52:32 +0000 Subject: [PATCH 131/256] Use depends_on (syntax: "depends-on modname") TCL modulefiles too. Also clean up the Lua generator for this a bit and fix the ignores of the unloads. --- easybuild/tools/module_generator.py | 46 +++++++++++++++++++---------- test/framework/module_generator.py | 14 +++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index a0cc1bf0b8..96e823d08b 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -405,6 +405,7 @@ class ModuleGeneratorTcl(ModuleGenerator): LOAD_REGEX = r"^\s*module\s+load\s+(\S+)" LOAD_TEMPLATE = "module load %(mod_name)s" + LOAD_TEMPLATE_DEPENDS_ON = "depends-on %(mod_name)s" def check_group(self, group, error_msg=None): """ @@ -505,9 +506,19 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body = [] if unload_modules: body.extend([self.unload_module(m).strip() for m in unload_modules]) - body.append(self.LOAD_TEMPLATE) + load_template = self.LOAD_TEMPLATE + # Lmod 7.6+ supports depends_on which does this most nicely: + if (build_option('recursive_mod_unload_depends_on') or + recursive_unload == 'depends_on'): + lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_depends_on_min = '7.6' + if (lmod_version != 'NOT_FOUND' and + LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + load_template = self.LOAD_TEMPLATE_DEPENDS_ON + body.append(load_template) - if build_option('recursive_mod_unload') or recursive_unload: + if (build_option('recursive_mod_unload') or recursive_unload or + load_template == self.LOAD_TEMPLATE_DEPENDS_ON): # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading; # when "module unload" is called on the module in which the dependency "module load" is present, # it will get translated to "module unload" @@ -670,6 +681,7 @@ class ModuleGeneratorLua(ModuleGenerator): LOAD_REGEX = r'^\s*load\("(\S+)"' LOAD_TEMPLATE = 'load("%(mod_name)s")' + LOAD_TEMPLATE_DEPENDS_ON = 'depends_on("%(mod_name)s")' PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")' UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)' @@ -795,8 +807,8 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body = [] if unload_modules: body.extend([self.unload_module(m).strip() for m in unload_modules]) - body.append(self.LOAD_TEMPLATE) + load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: if (build_option('recursive_mod_unload_depends_on') or recursive_unload == 'depends_on'): @@ -804,19 +816,23 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): lmod_depends_on_min = '7.6' if (lmod_version != 'NOT_FOUND' and LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): - return '\ndepends_on("%(mod_name)s")\n' % {'mod_name': mod_name} - - if build_option('recursive_mod_unload') or recursive_unload: - # wrapping the 'module load' with an 'is-loaded or mode == unload' - # guard ensures recursive unloading while avoiding load storms, - # when "module unload" is called on the module in which the - # depedency "module load" is present, it will get translated - # to "module unload" - # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html - load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"' + load_template = self.LOAD_TEMPLATE_DEPENDS_ON + + body.append(load_template) + if load_template == self.LOAD_TEMPLATE_DEPENDS_ON: + load_statement = body + [''] else: - load_guard = 'isloaded("%(mod_name)s")' - load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)] + if build_option('recursive_mod_unload') or recursive_unload: + # wrapping the 'module load' with an 'is-loaded or mode == unload' + # guard ensures recursive unloading while avoiding load storms, + # when "module unload" is called on the module in which the + # depedency "module load" is present, it will get translated + # to "module unload" + # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html + load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"' + else: + load_guard = 'isloaded("%(mod_name)s")' + load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)] return '\n'.join([''] + load_statement) % {'mod_name': mod_name} diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b342d57d15..c47dc9ed54 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -259,6 +259,20 @@ def test_load(self): init_config(build_options={'recursive_mod_unload': True}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + + # Lmod 7.6+ depends-on + lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_depends_on_min = '7.6' + if (lmod_version != 'NOT_FOUND' and + LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + expected = '\n'.join([ + '', + "depends-on mod_name", + '', + ]) + self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on")) + init_config(build_options={'recursive_mod_unload_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name")) else: # default: guarded module load (which implies no recursive unloading) expected = '\n'.join([ From 63bda19329fa52c7cb798d965c5ba73905bc7b7f Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 1 Feb 2018 17:48:04 +0000 Subject: [PATCH 132/256] Centralize Lmod version checks to modules_tool().has_depends_on/bump. We need to bump the minimum Lmod with TCL depends-on to 7.6.1. It's then easiest to just not care about 7.6.0, and make 7.6.1 the minimum supported Lmod for depends_on. --- .travis.yml | 4 ++-- easybuild/framework/easyconfig/default.py | 2 +- easybuild/tools/module_generator.py | 18 +++++------------- easybuild/tools/modules.py | 5 +++++ easybuild/tools/options.py | 2 +- test/framework/module_generator.py | 11 ++--------- test/framework/options.py | 6 +----- 7 files changed, 17 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98a362d46a..36008783c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ env: - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - LMOD_VERSION=6.6.3 - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - LMOD_VERSION=7.6 - - LMOD_VERSION=7.6 TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - LMOD_VERSION=7.6.1 + - LMOD_VERSION=7.6.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index e27feea11c..9e0c64ecb1 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -172,7 +172,7 @@ 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], - 'recursive_module_unload_depends_on' : [False, 'Use depends_on (Lmod 7.6+) for dependencies in generated module', MODULES], + 'recursive_module_unload_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module', MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 96e823d08b..335f2be92c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -507,14 +507,10 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): if unload_modules: body.extend([self.unload_module(m).strip() for m in unload_modules]) load_template = self.LOAD_TEMPLATE - # Lmod 7.6+ supports depends_on which does this most nicely: + # Lmod 7.6.1+ supports depends-on which does this most nicely: if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on'): - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') - lmod_depends_on_min = '7.6' - if (lmod_version != 'NOT_FOUND' and - LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): - load_template = self.LOAD_TEMPLATE_DEPENDS_ON + recursive_unload == 'depends_on') and modules_tool().has_depends_on: + load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) if (build_option('recursive_mod_unload') or recursive_unload or @@ -811,12 +807,8 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on'): - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') - lmod_depends_on_min = '7.6' - if (lmod_version != 'NOT_FOUND' and - LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): - load_template = self.LOAD_TEMPLATE_DEPENDS_ON + recursive_unload == 'depends_on') and modules_tool().has_depends_on: + load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) if load_template == self.LOAD_TEMPLATE_DEPENDS_ON: diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 62ae8db77c..6e7709b319 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -209,8 +209,11 @@ def modules(self): def set_and_check_version(self): """Get the module version, and check any requirements""" + lmod_depends_on_min = '7.6.1' if self.COMMAND in MODULE_VERSION_CACHE: self.version = MODULE_VERSION_CACHE[self.COMMAND] + self.has_depends_on = (isinstance(self, Lmod) and + StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min)) self.log.debug("Found cached version for %s: %s", self.COMMAND, self.version) return @@ -259,6 +262,8 @@ def set_and_check_version(self): self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION) MODULE_VERSION_CACHE[self.COMMAND] = self.version + self.has_depends_on = (isinstance(self, Lmod) and + StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min)) def check_cmd_avail(self): """Check whether modules tool command is available.""" diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9428d70eee..8a419b0ee3 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -474,7 +474,7 @@ def config_options(self): None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), - 'recursive-module-unload-depends-on': ("Use depends_on (Lmod 7.6+) for dependencies in all generated modules.", + 'recursive-module-unload-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index c47dc9ed54..5509729040 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -33,7 +33,6 @@ import sys import tempfile from unittest import TextTestRunner, TestSuite -from distutils.version import LooseVersion from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from vsc.utils.missing import nub @@ -261,10 +260,7 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name")) # Lmod 7.6+ depends-on - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') - lmod_depends_on_min = '7.6' - if (lmod_version != 'NOT_FOUND' and - LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + if self.modtool.has_depends_on: expected = '\n'.join([ '', "depends-on mod_name", @@ -299,10 +295,7 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name")) # Lmod 7.6+ depends_on - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') - lmod_depends_on_min = '7.6' - if (lmod_version != 'NOT_FOUND' and - LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + if self.modtool.has_depends_on: expected = '\n'.join([ '', 'depends_on("mod_name")', diff --git a/test/framework/options.py b/test/framework/options.py index e057469434..e43c2a167b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -35,7 +35,6 @@ import tempfile from unittest import TextTestRunner from urllib2 import URLError -from distutils.version import LooseVersion import easybuild.main import easybuild.tools.build_log @@ -1143,10 +1142,7 @@ def test_recursive_module_unload(self): # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') - lmod_depends_on_min = '7.6' - if (lmod_version != 'NOT_FOUND' and - LooseVersion(lmod_version) >= LooseVersion(lmod_depends_on_min)): + if self.modtool.has_depends_on: lastargs.append(lastargs[0]+'-depends-on') for lastarg in lastargs: args = [ From acf8cd185780e3fbfd900d46e35c825c0ed3df4b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 28 Feb 2018 20:16:00 +0000 Subject: [PATCH 133/256] Fix regular expression for module dependencies to match depends_on. --- easybuild/tools/module_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 335f2be92c..dc87e23b37 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -403,7 +403,7 @@ class ModuleGeneratorTcl(ModuleGenerator): MODULE_SHEBANG = '#%Module' CHARS_TO_ESCAPE = ['$'] - LOAD_REGEX = r"^\s*module\s+load\s+(\S+)" + LOAD_REGEX = r"^\s*module\s+(?:load|depends-on)\s+(\S+)" LOAD_TEMPLATE = "module load %(mod_name)s" LOAD_TEMPLATE_DEPENDS_ON = "depends-on %(mod_name)s" @@ -675,7 +675,7 @@ class ModuleGeneratorLua(ModuleGenerator): MODULE_SHEBANG = '' # no 'shebang' in Lua module files CHARS_TO_ESCAPE = [] - LOAD_REGEX = r'^\s*load\("(\S+)"' + LOAD_REGEX = r'^\s*(?:load|depends_on)\("(\S+)"' LOAD_TEMPLATE = 'load("%(mod_name)s")' LOAD_TEMPLATE_DEPENDS_ON = 'depends_on("%(mod_name)s")' From 15c9d4a7b33c3a4bb79aa32d6f0b37265a95f628 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 1 Mar 2018 22:57:58 +0100 Subject: [PATCH 134/256] fix target account for --update-pr in case it's different from GitHub account being used to push branch --- easybuild/tools/github.py | 12 ++++++++---- test/framework/options.py | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c4c035d8e5..ccab6cc6d0 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -769,15 +769,19 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target git_repo.index.commit(commit_msg) # push to GitHub - github_account = build_option('github_org') or build_option('github_user') - github_url = 'git@github.com:%s/%s.git' % (github_account, pr_target_repo) + github_url = 'git@github.com:%s/%s.git' % (target_account, pr_target_repo) salt = ''.join(random.choice(string.letters) for _ in range(5)) - remote_name = 'github_%s_%s' % (github_account, salt) + remote_name = 'github_%s_%s' % (target_account, salt) dry_run = build_option('dry_run') or build_option('extended_dry_run') + push_branch_msg = "pushing branch '%s' to remote '%s' (%s)" % (pr_branch, remote_name, github_url) + if dry_run: + push_branch_msg += ' [DRY RUN]' + print_msg(push_branch_msg) + if not dry_run: - _log.debug("Pushing branch '%s' to remote '%s' (%s)", pr_branch, remote_name, github_url) + _log.debug(push_log_msg) try: my_remote = git_repo.create_remote(remote_name, github_url) res = my_remote.push(pr_branch) diff --git a/test/framework/options.py b/test/framework/options.py index 13e5ee09f6..fddc97393b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2585,6 +2585,7 @@ def test_new_update_pr(self): r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", r"^\s*1 file changed", + "^== pushing branch 'develop' to remote '.*' \(git@github.com:easybuilders/easybuild-easyconfigs.git\)", r"^Updated easybuilders/easybuild-easyconfigs PR #2237 by pushing to branch easybuilders/develop \[DRY RUN\]", ] for regex in regexs: From e3f0b20b17d24237f03c5e339cb8d6147226be51 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 07:36:36 +0100 Subject: [PATCH 135/256] temporarily disable test suite runs for environment modules --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20b4f1ce77..bc6887681a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,9 @@ env: - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - LMOD_VERSION=7.4 - LMOD_VERSION=7.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl + #- ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules + #- ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules matrix: # mark build as finished as soon as job has failed fast_finish: true From 37081ce67eb1c1f0e9fadf5c9e4c8988a0a3156e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 08:53:59 +0100 Subject: [PATCH 136/256] fix broken debug log statement --- easybuild/tools/github.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ccab6cc6d0..e2a94b5084 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -778,10 +778,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target push_branch_msg = "pushing branch '%s' to remote '%s' (%s)" % (pr_branch, remote_name, github_url) if dry_run: push_branch_msg += ' [DRY RUN]' - print_msg(push_branch_msg) + print_msg(push_branch_msg, log=_log) if not dry_run: - _log.debug(push_log_msg) try: my_remote = git_repo.create_remote(remote_name, github_url) res = my_remote.push(pr_branch) From 0f84599046d44c6f1dc70e2817b735fe0ed38fdb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 09:08:28 +0100 Subject: [PATCH 137/256] clean up dry run logic when pushing branch --- easybuild/tools/github.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e2a94b5084..278dc94034 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -777,10 +777,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target push_branch_msg = "pushing branch '%s' to remote '%s' (%s)" % (pr_branch, remote_name, github_url) if dry_run: - push_branch_msg += ' [DRY RUN]' - print_msg(push_branch_msg, log=_log) - - if not dry_run: + print_msg(push_branch_msg + '[DRY RUN]', log=_log) + else: + print_msg(push_branch_msg, log=_log) try: my_remote = git_repo.create_remote(remote_name, github_url) res = my_remote.push(pr_branch) From fdfc358d6aaede5a74477dfe6a0d80f69db3eef9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 09:14:58 +0100 Subject: [PATCH 138/256] also include PR #2419 in EasyBuild v3.5.2 release notes --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 17149a65a0..0d85f9bec4 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -18,6 +18,7 @@ bugfix release - update bootstrap script to be compatible with Modules v4 (#2390) - avoid fatal error when determining glibc version on non-glibc Linux system (e.g. Alphine Linux) (#2398) - exclude location of RPATH wrappers from $PATH to avoid fork bomb (#2410) + - fix target account for --update-pr in case it's different from GitHub account being used to push branch (#2419) v3.5.1 (January 16th 2018) From 3876c69726f196361017a37159c787ca3589352f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 10:54:28 +0100 Subject: [PATCH 139/256] add missing space --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 278dc94034..e1d65e0133 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -777,7 +777,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target push_branch_msg = "pushing branch '%s' to remote '%s' (%s)" % (pr_branch, remote_name, github_url) if dry_run: - print_msg(push_branch_msg + '[DRY RUN]', log=_log) + print_msg(push_branch_msg + ' [DRY RUN]', log=_log) else: print_msg(push_branch_msg, log=_log) try: From 17b1d8ca3db09a71c6e32fcec3ee8e068183ea75 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 11:52:13 +0100 Subject: [PATCH 140/256] fix order in result of gen_list_easyblocks and gen_easyblocks_overview_rst --- easybuild/tools/docs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 741ad83ed4..c428a9eb19 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -490,7 +490,7 @@ def add_class(classes, cls): children = cls.__subclasses__() classes.update({cls.__name__: { 'module': cls.__module__, - 'children': [x.__name__ for x in children] + 'children': sorted([x.__name__ for x in children], key=lambda x: x.lower()) }}) for child in children: add_class(classes, child) @@ -891,12 +891,14 @@ def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc if eb_class.extra_options(): title = 'Extra easyconfig parameters specific to ``%s`` easyblock' % classname ex_opt = eb_class.extra_options() + keys = sorted(ex_opt.keys()) + values = [ex_opt[k] for k in keys] table_titles = ['easyconfig parameter', 'description', 'default value'] table_values = [ - ['``' + key + '``' for key in ex_opt], # parameter name - [val[1] for val in ex_opt.values()], # description - ['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()] # default value + ['``' + key + '``' for key in keys], # parameter name + [val[1] for val in values], # description + ['``' + str(quote_str(val[0])) + '``' for val in values] # default value ] doc.extend(rst_title_and_table(title, table_titles, table_values)) From c1af5e25e06c605bea2e47bf5e672550c3846375 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 19:22:35 +0100 Subject: [PATCH 141/256] re-enable testing against environment modules, bump Lmod to 7.7.16 --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc6887681a..6efc262860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,13 +5,11 @@ env: # purposely specifying slowest builds first, to gain time overall - LMOD_VERSION=5.8 - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - LMOD_VERSION=6.6.3 - - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - LMOD_VERSION=7.4 - - LMOD_VERSION=7.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - #- ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - LMOD_VERSION=7.7.16 + - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - #- ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules + - ENV_MOD_VERSION=4.1.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules matrix: # mark build as finished as soon as job has failed fast_finish: true From 4e7125d7fea651ed34841acb215dbd50cf79b18c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 2 Mar 2018 19:23:35 +0100 Subject: [PATCH 142/256] bump version to v3.6.0dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 99a8507f91..e98d0b8b7d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.2') +VERSION = LooseVersion('3.6.0.dev0') UNKNOWN = 'UNKNOWN' def get_git_revision(): From c55a83af25c105b7dcd6e065c0b3386c92387a42 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 4 Mar 2018 16:55:22 +0100 Subject: [PATCH 143/256] stick with Environment Modules 4.0.0 for now --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6efc262860..e7970b2297 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - ENV_MOD_VERSION=4.1.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules + - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules matrix: # mark build as finished as soon as job has failed fast_finish: true From b587bce5a73056224c96f2949b9a3df54b474b7a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 4 Mar 2018 20:30:44 +0100 Subject: [PATCH 144/256] enhance test for --list-easyblock to also check output order --- test/framework/options.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index fddc97393b..9d90f47758 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -551,14 +551,29 @@ def test_list_easyblocks(self): self.eb_main(args, logfile=dummylogfn) logtxt = read_file(self.logfile) - for pat in [ - r"EasyBlock\n", - r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n", - r"|--\s+bar\n", - ]: - - msg = "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, logtxt) - self.assertTrue(re.search(pat, logtxt), msg) + expected = '\n'.join([ + r'EasyBlock', + r'\|-- bar', + r'\|-- ConfigureMake', + r'\|-- EB_foo', + r'\| \|-- EB_foofoo', + r'\|-- EB_GCC', + r'\|-- EB_HPL', + r'\|-- EB_ScaLAPACK', + r'\|-- EB_toy_buggy', + r'\|-- ExtensionEasyBlock', + r'\| \|-- DummyExtension', + r'\| \|-- EB_toy', + r'\| \|-- Toy_Extension', + r'\|-- Toolchain', + r'Extension', + r'\|-- ExtensionEasyBlock', + r'\| \|-- DummyExtension', + r'\| \|-- EB_toy', + r'\| \|-- Toy_Extension', + ]) + regex = re.compile(expected, re.M) + self.assertTrue(regex.search(logtxt), "Pattern '%s' found in: %s" % (regex.pattern, logtxt)) # clear log write_file(self.logfile, '') From 7e329e5871c143cadbc820bdbcdb20e82445abf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 4 Mar 2018 20:51:58 +0100 Subject: [PATCH 145/256] extend test for gen_easyblocks_overview_rst to check order of custom easyconfig parameters --- test/framework/docs.py | 13 ++++++++++++- .../easybuild/easyblocks/generic/configuremake.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 29c2cd590c..390c3addaf 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -62,6 +62,17 @@ def test_gen_easyblocks(self): '', "Dummy support for building and installing applications with configure/make/make install.", '', + "Extra easyconfig parameters specific to ``ConfigureMake`` easyblock", + "-------------------------------------------------------------------", + '', + "==================== ============ =============", + "easyconfig parameter description default value", + "==================== ============ =============", + '``test_123`` Test 1, 2, 3 ``""`` ', + "``test_bool`` Just a test ``False`` ", + "``test_none`` Another test ``None`` ", + "==================== ============ =============", + '', "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", "--------------------------------------------------------------------", '', @@ -74,7 +85,7 @@ def test_gen_easyblocks(self): "==================== ================================================================", ]) - self.assertTrue(check_configuremake in ebdoc) + self.assertTrue(check_configuremake in ebdoc, "Found '%s' in: %s" % (check_configuremake, ebdoc)) names = [] for mod in modules: diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 64c1a38d99..bb73fdfeae 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -29,7 +29,18 @@ @author: Kenneth Hoste (Ghent University) """ from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM class ConfigureMake(EasyBlock): """Dummy support for building and installing applications with configure/make/make install.""" - pass + + @staticmethod + def extra_options(extra_vars=None): + """Extra easyconfig parameters specific to ConfigureMake.""" + extra_vars = EasyBlock.extra_options(extra=extra_vars) + extra_vars.update({ + 'test_bool': [False, "Just a test", CUSTOM], + 'test_none': [None, "Another test", CUSTOM], + 'test_123': ['', "Test 1, 2, 3", CUSTOM], + }) + return extra_vars From 57bf5ec7c9efa6ba375201a63c267f36321ce496 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Mar 2018 15:13:03 +0100 Subject: [PATCH 146/256] fix target account for branch pushed when using --new-pr --- easybuild/tools/github.py | 20 ++++++++++++++------ test/framework/options.py | 2 ++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e1d65e0133..a0bf60a6a5 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -621,7 +621,7 @@ def setup_repo(git_repo, target_account, target_repo, branch_name, silent=False, @only_if_module_is_available('git', pkgname='GitPython') -def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target_account=None, commit_msg=None): +def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_account=None, commit_msg=None): """ Common code for new_pr and update_pr functions: * check whether all supplied paths point to existing files @@ -633,9 +633,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) :param ecs: list of parsed easyconfigs, incl. for dependencies (if robot is enabled) - :param start_branch: name of branch to start from + :param start_branch: name of branch to use as base for PR :param pr_branch: name of branch to push to GitHub - :param target_account: name of target GitHub account for PR + :param start_account: name of GitHub account to use as base for PR :param commit_msg: commit message to use """ # we need files to create the PR with @@ -665,10 +665,18 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, target raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo) if start_branch is None: + # if start branch is not specified, we're opening a new PR + # account to use is determined by active EasyBuild configuration (--github-org or --github-user) + target_account = build_option('github_org') or build_option('github_user') + # if branch to start from is specified, we're updating an existing PR start_branch = build_option('pr_target_branch') + else: + # account to target is the one that owns the branch used to open PR + # (which may be different from account used to push update!) + target_account = start_account # set up repository - setup_repo(git_repo, target_account, pr_target_repo, start_branch) + setup_repo(git_repo, start_account, pr_target_repo, start_branch) _log.debug("git status: %s", git_repo.git.status()) @@ -1060,7 +1068,7 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): # create branch, commit files to it & push to GitHub file_info, deleted_paths, git_repo, branch, diff_stat = _easyconfigs_pr_common(paths, ecs, pr_branch=pr_branch_name, - target_account=pr_target_account, + start_account=pr_target_account, commit_msg=commit_msg) # only use most common toolchain(s) in toolchain label of PR title @@ -1160,7 +1168,7 @@ def update_pr(pr, paths, ecs, commit_msg=None): print_msg("Determined branch name corresponding to %s PR #%s: %s" % (github_target, pr, branch), log=_log) _, _, _, _, diff_stat = _easyconfigs_pr_common(paths, ecs, start_branch=branch, pr_branch=branch, - target_account=account, commit_msg=commit_msg) + start_account=account, commit_msg=commit_msg) print_msg("Overview of changes:\n%s\n" % diff_stat, log=_log, prefix=False) diff --git a/test/framework/options.py b/test/framework/options.py index fddc97393b..898bc1d05f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2458,8 +2458,10 @@ def test_new_update_pr(self): else: self.assertTrue(False, "Failed to find temporary git working dir: %s" % dirs) + remote = 'git@github.com:%s/easybuild-easyconfigs.git' % GITHUB_TEST_ACCOUNT regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", + r"^== pushing branch '.*' to remote '.*' \(%s\)" % remote, r"^Opening pull request \[DRY RUN\]", r"^\* target: easybuilders/easybuild-easyconfigs:develop", r"^\* from: %s/easybuild-easyconfigs:.*_new_pr_toy00" % GITHUB_TEST_ACCOUNT, From 8fd36f95d081e81ab85e4e186df3adf507bbe1ee Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Tue, 6 Mar 2018 11:32:45 +0100 Subject: [PATCH 147/256] Print which hook is being executed in the command line output --- easybuild/tools/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index d379aeeb6e..606c8b5276 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -32,7 +32,7 @@ import os from vsc.utils import fancylogger -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_msg _log = fancylogger.getLogger('hooks', fname=False) @@ -172,5 +172,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None) if args is None: args = [] - _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) + print_msg("Running %s hook (arguments: %s)..." % (hook.__name__, args), log=_log) hook(*args) From 34571bb5bcacb04553f6401e1932e1488f2a7cf1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Mar 2018 18:41:16 +0100 Subject: [PATCH 148/256] only print hook name, fix test for run_hook --- easybuild/tools/hooks.py | 3 ++- test/framework/hooks.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 606c8b5276..a0d3dc332b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -172,5 +172,6 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None) if args is None: args = [] - print_msg("Running %s hook (arguments: %s)..." % (hook.__name__, args), log=_log) + print_msg("Running %s hook..." % hook.__name__) + _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) hook(*args) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 075020a526..5677a8a567 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -116,9 +116,12 @@ def test_run_hook(self): self.mock_stderr(False) expected_stdout = '\n'.join([ + "== Running start_hook hook...", "running start hook", + "== Running post_configure_hook hook...", "running post-configure hook", "running foo helper method", + "== Running pre_install_hook hook...", "running pre-install hook", ]) From e43bec08a9eab4bdab76ebe4dc69f56ba1321628 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Mar 2018 18:48:14 +0100 Subject: [PATCH 149/256] prettier printed output for triggered hooks --- easybuild/tools/hooks.py | 9 +++++++-- test/framework/hooks.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index a0d3dc332b..5da7ef2fa2 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -172,6 +172,11 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None) if args is None: args = [] - print_msg("Running %s hook..." % hook.__name__) - _log.info("Running %s hook (arguments: %s)...", hook.__name__, args) + if pre_step_hook: + label = 'pre-' + label + elif post_step_hook: + label = 'post-' + label + + print_msg("Running %s hook..." % label) + _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) hook(*args) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 5677a8a567..cf53389a10 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -46,17 +46,17 @@ def setUp(self): self.test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py') test_hooks_pymod_txt = '\n'.join([ 'def start_hook():', - ' print("running start hook")', + ' print("this is triggered at the very beginning")', '', 'def foo():', ' print("running foo helper method")', '', 'def post_configure_hook(self):', - ' print("running post-configure hook")', + ' print("this is run after configure step")', ' foo()', '', 'def pre_install_hook(self):', - ' print("running pre-install hook")', + ' print("this is run before install step")', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -116,13 +116,13 @@ def test_run_hook(self): self.mock_stderr(False) expected_stdout = '\n'.join([ - "== Running start_hook hook...", - "running start hook", - "== Running post_configure_hook hook...", - "running post-configure hook", + "== Running start hook...", + "this is triggered at the very beginning", + "== Running post-configure hook...", + "this is run after configure step", "running foo helper method", - "== Running pre_install_hook hook...", - "running pre-install hook", + "== Running pre-install hook...", + "this is run before install step", ]) self.assertEqual(stdout.strip(), expected_stdout) From 23525e59e915de435125e32fdfee0371e0222a13 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Mar 2018 20:05:47 +0100 Subject: [PATCH 150/256] also fix test_toy_build_hooks --- test/framework/toy_build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6967e18b68..fe3d442bb5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1807,11 +1807,16 @@ def test_toy_build_hooks(self): self.assertEqual(stderr, '') expected_output = '\n'.join([ + "== Running start hook...", "start hook triggered", + "== Running pre-configure hook...", "pre-configure: toy.source: True", + "== Running post-configure hook...", "post-configure: toy.source: False", + "== Running post-install hook...", "in post-install hook for toy v0.0", "bin, lib", + "== Running end hook...", "end hook triggered, all done!", ]) self.assertEqual(stdout.strip(), expected_output) From 99e9f89602fd28900dee8fb7deac369593b6bcf9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Mar 2018 10:30:15 +0100 Subject: [PATCH 151/256] force tests for --include-easyblocks to run last, to avoid breaking the --list-easyblocks test --- test/framework/options.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9d90f47758..a9c7df11ca 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1907,7 +1907,9 @@ def generate_cmd_line(ebopts): ebopts = EasyBuildOptions(go_args=args, envvar_prefix='EASYBUILD') self.assertEqual(generate_cmd_line(ebopts), expected) - def test_include_easyblocks(self): + # must be run after test for --list-easyblocks, hence the '_xxx_' + # cleaning up the imported easyblocks is quite difficult... + def test_xxx_include_easyblocks(self): """Test --include-easyblocks.""" orig_local_sys_path = sys.path[:] @@ -1969,7 +1971,9 @@ def test_include_easyblocks(self): # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.foo'] - def test_include_generic_easyblocks(self): + # must be run after test for --list-easyblocks, hence the '_xxx_' + # cleaning up the imported easyblocks is quite difficult... + def test_xxx_include_generic_easyblocks(self): """Test --include-easyblocks with a generic easyblock.""" orig_local_sys_path = sys.path[:] fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From 96d981439b9c6d0786360df8bec26d02113d05a7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Mar 2018 11:56:40 +0100 Subject: [PATCH 152/256] fix tiny remark --- easybuild/tools/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index c428a9eb19..8bdb2338da 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -490,7 +490,7 @@ def add_class(classes, cls): children = cls.__subclasses__() classes.update({cls.__name__: { 'module': cls.__module__, - 'children': sorted([x.__name__ for x in children], key=lambda x: x.lower()) + 'children': sorted([c.__name__ for c in children], key=lambda x: x.lower()) }}) for child in children: add_class(classes, child) From 149dd6e468ac563dddcb4c75b87e03009a7e4af6 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 7 Mar 2018 20:53:54 +0800 Subject: [PATCH 153/256] prepare release notes for eb353 --- RELEASE_NOTES | 12 ++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 0d85f9bec4..3713858761 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,18 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v3.5.3 (March 7th 2018) +----------------------- + +bugfix release +- various enhancements, including: + - re-enable testing against environment modules, bump Lmod to 7.7.16 (#2425) + - print which hook is being executed in the command line output (#2427) +- various bug fixes, including: + - fix order in result of gen_list_easyblocks and gen_easyblocks_overview_rst (#2421) + - fix target account for branch pushed when using --new-pr (#2426) + + v3.5.2 (March 2nd 2018) ----------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e98d0b8b7d..dfbf300910 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.6.0.dev0') +VERSION = LooseVersion('3.5.3') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 39cb3bb2e040824d8e1258a67e0fbe46bcfba373 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Mar 2018 18:00:40 +0100 Subject: [PATCH 154/256] bump version to v3.6.0dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index dfbf300910..e98d0b8b7d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.5.3') +VERSION = LooseVersion('3.6.0.dev0') UNKNOWN = 'UNKNOWN' def get_git_revision(): From 63eef2bc72512dcdfec9e6730cc495105c67a140 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 8 Mar 2018 11:33:17 +0100 Subject: [PATCH 155/256] determine whether included easyblocks are generic or not based on class they define --- easybuild/tools/include.py | 16 ++++++++++------ test/framework/include.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index b7d15655ee..c69bff78a7 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -29,11 +29,12 @@ :author: Kenneth Hoste (Ghent University) """ import os +import re import sys from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import expand_glob_paths, symlink +from easybuild.tools.filetools import expand_glob_paths, read_file, symlink # these are imported just to we can reload them later import easybuild.tools.module_naming_scheme import easybuild.toolchains @@ -142,6 +143,11 @@ def verify_imports(pymods, pypkg, from_path): _log.debug("Import of %s from %s verified", pymod_spec, from_path) +def is_software_specific_easyblock(module): + """Determine whether Python module at specified location is a software-specific easyblock.""" + return bool(re.search('^class EB_.*\(.*\):$', read_file(module), re.M)) + + def include_easyblocks(tmpdir, paths): """Include generic and software-specific easyblocks found in specified locations.""" easyblocks_path = os.path.join(tmpdir, 'included-easyblocks') @@ -155,12 +161,10 @@ def include_easyblocks(tmpdir, paths): for easyblock_module in allpaths: filename = os.path.basename(easyblock_module) - # generic easyblocks are expected to be in a directory named 'generic' - parent_dir = os.path.basename(os.path.dirname(easyblock_module)) - if parent_dir == 'generic': - target_path = os.path.join(easyblocks_dir, 'generic', filename) - else: + if is_software_specific_easyblock(easyblock_module): target_path = os.path.join(easyblocks_dir, filename) + else: + target_path = os.path.join(easyblocks_dir, 'generic', filename) if not os.path.exists(target_path): symlink(easyblock_module, target_path) diff --git a/test/framework/include.py b/test/framework/include.py index 7e50df53f6..881f737eea 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -32,8 +32,10 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains +from easybuild.tools.include import is_software_specific_easyblock def up(path, cnt): @@ -232,6 +234,20 @@ def test_include_toolchains(self): my_tc_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_tc_pyc_path), 'my_tc.py')) self.assertTrue(os.path.samefile(up(my_tc_real_py_path, 1), my_toolchains)) + def test_is_software_specific_easyblock(self): + """Test is_software_specific_easyblock function.""" + + self.assertErrorRegex(EasyBuildError, "No such file", is_software_specific_easyblock, '/no/such/easyblock.py') + + testdir = os.path.dirname(os.path.abspath(__file__)) + test_easyblocks = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') + + self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 'g', 'gcc.py'))) + self.assertTrue(is_software_specific_easyblock(os.path.join(test_easyblocks, 't', 'toy.py'))) + + self.assertFalse(is_software_specific_easyblock(os.path.join(test_easyblocks, 'generic', 'configuremake.py'))) + self.assertFalse(is_software_specific_easyblock(os.path.join(test_easyblocks, 'generic', 'toolchain.py'))) + def suite(): """ returns all the testcases in this module """ From f0a33a1b50070d7c24e262bc390cfebea1448ca8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Mar 2018 08:36:16 +0100 Subject: [PATCH 156/256] fix checking for GenericTest (generic) easyblock in test --- test/framework/options.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index ac8c80a323..f46f33c1cb 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2040,16 +2040,17 @@ def test_xxx_include_generic_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) + mod_pattern = 'easybuild.easyblocks.generic.generictest' path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', - 'generictest.py') - foo_regex = re.compile(r"^\|-- GenericTest \(easybuild.easyblocks.generictest @ %s\)" % path_pattern, re.M) + 'generic', 'generictest.py') + foo_regex = re.compile(r"^\|-- GenericTest \(%s @ %s\)" % (mod_pattern, path_pattern), re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) klass = get_easyblock_class('GenericTest') self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) # 'undo' import of foo easyblock - del sys.modules['easybuild.easyblocks.generictest'] + del sys.modules['easybuild.easyblocks.generic.generictest'] def test_include_module_naming_schemes(self): """Test --include-module-naming-schemes.""" From 7fc274121605ef09ab6baf61736a7ce8ca21de96 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sat, 10 Mar 2018 20:36:04 +0100 Subject: [PATCH 157/256] fix checksum lines > 120 chars --- easybuild/framework/easyblock.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 68eb4cc91e..afbd40bec7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2954,7 +2954,12 @@ def inject_checksums(ecs, checksum_type): else: checksum_lines = ['checksums = ['] for fn, checksum in checksums: - checksum_lines.append("%s'%s', # %s" % (INDENT_4SPACES, checksum, fn)) + check_line = "%s'%s', # %s" % (INDENT_4SPACES, checksum, fn) + if len(check_line) > 120: + checksum_lines.append("%s# %s" % (INDENT_4SPACES, fn)) + checksum_lines.append("%s'%s'," % (INDENT_4SPACES, checksum)) + else: + checksum_lines.append(check_line) checksum_lines.append(']\n') checksums_txt = '\n'.join(checksum_lines) From 9586e1721b8aff4c4afc15a239a85db942a29946 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Mar 2018 10:29:14 +0100 Subject: [PATCH 158/256] enable caching of $HOME/.cache/pip in Travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index e7970b2297..1fb7ed6ec6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ env: - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules +cache: + pip: true matrix: # mark build as finished as soon as job has failed fast_finish: true From 5782636094e9eeae427e91c732978ae23a2a32c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Mar 2018 15:55:51 +0100 Subject: [PATCH 159/256] also avoid adding too long checksum linkes for extensions + use constant for 120 chars limit --- easybuild/framework/easyblock.py | 13 +++++++++++-- easybuild/framework/easyconfig/style.py | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index afbd40bec7..40b64c4b71 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -58,6 +58,7 @@ from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template from easybuild.framework.easyconfig.format.format import INDENT_4SPACES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig +from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP from easybuild.tools.build_details import get_build_stats @@ -2955,7 +2956,7 @@ def inject_checksums(ecs, checksum_type): checksum_lines = ['checksums = ['] for fn, checksum in checksums: check_line = "%s'%s', # %s" % (INDENT_4SPACES, checksum, fn) - if len(check_line) > 120: + if len(check_line) > MAX_LINE_LENGTH: checksum_lines.append("%s# %s" % (INDENT_4SPACES, fn)) checksum_lines.append("%s'%s'," % (INDENT_4SPACES, checksum)) else: @@ -3050,7 +3051,15 @@ def inject_checksums(ecs, checksum_type): else: exts_list_lines.append("%s'checksums': [" % (INDENT_4SPACES * 2)) for fn, checksum in ext_checksums: - exts_list_lines.append("%s'%s', # %s" % (INDENT_4SPACES * 3, checksum, fn)) + line_indent = INDENT_4SPACES * 3 + ext_checksum_line = "%s'%s', # %s" % (line_indent, checksum, fn) + if len(ext_checksum_line) > MAX_LINE_LENGTH: + exts_list_lines.extend([ + "%s# %s" % (line_indent, fn), + "%s'%s'," % (line_indent, checksum), + ]) + else: + exts_list_lines.append(ext_checksum_line) exts_list_lines.append("%s]," % (INDENT_4SPACES * 2)) if ext_options or ext_checksums: diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index fc325550f7..e485dd260b 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -53,6 +53,8 @@ COMMENT_REGEX = re.compile(r'^\s*#') PARAM_DEF_REGEX = re.compile(r"^(?P[a-z_]+)\s*=\s*") +MAX_LINE_LENGTH = 120 + # Any function starting with _eb_check_ (see EB_CHECK variable) will be # added to the tests if the test number is added to the select list. @@ -126,7 +128,7 @@ def check_easyconfigs_style(easyconfigs, verbose=False): options = styleguide.options # we deviate from standard pep8 and allow 120 chars # on a line: the default of 79 is too narrow. - options.max_line_length = 120 + options.max_line_length = MAX_LINE_LENGTH # we ignore some tests # note that W291 has been replaced by our custom W299 options.ignore = ( From 0249dbbf999cd31b8508f19a18b18cb58011fdab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 11 Mar 2018 16:13:12 +0100 Subject: [PATCH 160/256] enhance test for --inject-checksums by using longer names for test patch files --- easybuild/framework/easyblock.py | 6 ++-- test/framework/easyblock.py | 17 ++++----- test/framework/easyconfig.py | 10 +++--- .../test_ecs/t/toy/toy-0.0-deps.eb | 2 +- .../t/toy/toy-0.0-gompi-1.3.12-test.eb | 4 +-- .../test_ecs/t/toy/toy-0.0-iter.eb | 2 +- .../test_ecs/t/toy/toy-0.0-multiple.eb | 2 +- .../easyconfigs/test_ecs/t/toy/toy-0.0.eb | 2 +- test/framework/easyconfigs/yeb/toy-0.0.yeb | 2 +- test/framework/filetools.py | 14 +++++--- test/framework/github.py | 2 +- test/framework/options.py | 35 +++++++++++++------ ..._fix-silly-typo-in-printf-statement.patch} | 0 ..._fix-silly-typo-in-printf-statement.patch} | 0 test/framework/toy_build.py | 6 ++-- 15 files changed, 63 insertions(+), 41 deletions(-) rename test/framework/sandbox/sources/toy/extensions/{bar-0.0_typo.patch => bar-0.0_fix-silly-typo-in-printf-statement.patch} (100%) rename test/framework/sandbox/sources/toy/{toy-0.0_typo.patch => toy-0.0_fix-silly-typo-in-printf-statement.patch} (100%) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 40b64c4b71..4d0314d60e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2957,8 +2957,10 @@ def inject_checksums(ecs, checksum_type): for fn, checksum in checksums: check_line = "%s'%s', # %s" % (INDENT_4SPACES, checksum, fn) if len(check_line) > MAX_LINE_LENGTH: - checksum_lines.append("%s# %s" % (INDENT_4SPACES, fn)) - checksum_lines.append("%s'%s'," % (INDENT_4SPACES, checksum)) + checksum_lines.extend([ + "%s# %s" % (INDENT_4SPACES, fn), + "%s'%s'," % (INDENT_4SPACES, checksum), + ]) else: checksum_lines.append(check_line) checksum_lines.append(']\n') diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index f9af775402..e7f80961c8 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -847,27 +847,28 @@ def test_fetch_patches(self): eb.fetch_patches() self.assertEqual(len(eb.patches), 2) - self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[0]['name'], 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertFalse('level' in eb.patches[0]) # reset eb.patches = [] + toy_patch = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' patches = [ - ('toy-0.0_typo.patch', 0), # should also be level 0 (not None or something else) - ('toy-0.0_typo.patch', 4), # should be level 4 - ('toy-0.0_typo.patch', 'foobar'), # sourcepath should be set to 'foobar' + (toy_patch, 0), # should also be level 0 (not None or something else) + (toy_patch, 4), # should be level 4 + (toy_patch, 'foobar'), # sourcepath should be set to 'foobar' ('toy-0.0.tar.gz', 'some/path'), # copy mode (not a .patch file) ] # check if patch levels are parsed correctly eb.fetch_patches(patch_specs=patches) self.assertEqual(len(eb.patches), 4) - self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[0]['name'], toy_patch) self.assertEqual(eb.patches[0]['level'], 0) - self.assertEqual(eb.patches[1]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[1]['name'], toy_patch) self.assertEqual(eb.patches[1]['level'], 4) - self.assertEqual(eb.patches[2]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[2]['name'], toy_patch) self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), self.assertEqual(eb.patches[3]['copy'], 'some/path') @@ -1061,7 +1062,7 @@ def test_patch_step(self): orig_sources = ec['ec']['sources'][:] toy_patches = [ - 'toy-0.0_typo.patch', # test for applying patch + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', # test for applying patch ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy ] self.assertEqual(ec['ec']['patches'], toy_patches) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 6feeacc38c..61a787c2bb 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -323,7 +323,7 @@ def test_exts_list(self): # SHA256 checksum for source (gzip-1.4.eb) ' "6f281b6d7a3965476324a23b9d80232bd4ffe3967da85e4b7c01d9d81d649a09",', # SHA256 checksum for 'patch' (toy-0.0.eb) - ' "044e300a051120defb01c14c7c06e9aa4bca40c5d589828df360e2684dcc9074",', + ' "a79ba0ef5dceb5b8829268247feae8932bed2034c6628ff1d92c84bf45e9a546",', ' ],', ' }),', ']', @@ -344,7 +344,7 @@ def test_exts_list(self): self.assertEqual(exts_sources[1]['version'], '2.0') self.assertEqual(exts_sources[1]['options'], { 'checksums': ['6f281b6d7a3965476324a23b9d80232bd4ffe3967da85e4b7c01d9d81d649a09', - '044e300a051120defb01c14c7c06e9aa4bca40c5d589828df360e2684dcc9074'], + 'a79ba0ef5dceb5b8829268247feae8932bed2034c6628ff1d92c84bf45e9a546'], 'patches': ['toy-0.0.eb'], 'source_tmpl': 'gzip-1.4.eb', 'source_urls': [('http://example.com', 'suffix')], @@ -1349,7 +1349,8 @@ def test_update(self): # for list values: extend ec.update('patches', ['foo.patch', 'bar.patch']) - self.assertEqual(ec['patches'], ['toy-0.0_typo.patch', ('toy-extra.txt', 'toy-0.0'), 'foo.patch', 'bar.patch']) + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + self.assertEqual(ec['patches'], [toy_patch_fn, ('toy-extra.txt', 'toy-0.0'), 'foo.patch', 'bar.patch']) def test_hide_hidden_deps(self): """Test use of --hide-deps on hiddendependencies.""" @@ -2009,7 +2010,8 @@ def test_categorize_files_by_type(self): self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': []}, categorize_files_by_type([])) test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) - toy_patch = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'sources', 'toy', toy_patch_fn) paths = [ 'bzip2-1.0.6.eb', os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb index 62ab5d8f40..f2b34b5b00 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-deps.eb @@ -17,7 +17,7 @@ checksums = [[ ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), ('size', 273), ]] -patches = ['toy-0.0_typo.patch'] +patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] dependencies = [ ('ictce', '4.1.13', '', True), diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb index ef9585dde6..bb01473b53 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-1.3.12-test.eb @@ -18,7 +18,7 @@ checksums = [[ ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), ('size', 273), ]] -patches = ['toy-0.0_typo.patch'] +patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] exts_default_options = { 'source_urls': ['http://example.com/%(name)s'], @@ -30,7 +30,7 @@ exts_list = [ 'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'], # checksum for # custom extension filter to verify use of stdin value being passed to filter command 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), - 'patches': ['bar-0.0_typo.patch'], + 'patches': ['bar-0.0_fix-silly-typo-in-printf-statement.patch'], 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', }), diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index b9fd8ec86a..fe6e2840f5 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -8,7 +8,7 @@ description = "Toy C program, 100% toy." toolchain = {'name': 'dummy', 'version': 'dummy'} sources = [SOURCE_TAR_GZ] -patches = ['toy-0.0_typo.patch'] +patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] buildopts = [ '', diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb index 7318c32935..abb90dae56 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-multiple.eb @@ -9,7 +9,7 @@ description = "Toy C program, 100% toy." toolchain = {'name': 'dummy', 'version': 'dummy'} sources = [SOURCE_TAR_GZ] -patches = ['toy-0.0_typo.patch'] +patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] checksums = [ ('adler32', '0x998410035'), 'e6785e1a721fc8bf79892e3ef41557c0', diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb index bdcd5aca58..b8a1ad8c3d 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0.eb @@ -17,7 +17,7 @@ checksums = [[ ('size', 273), ]] patches = [ - 'toy-0.0_typo.patch', + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', ('toy-extra.txt', 'toy-0.0'), ] diff --git a/test/framework/easyconfigs/yeb/toy-0.0.yeb b/test/framework/easyconfigs/yeb/toy-0.0.yeb index f6762078b4..0b0b668ed3 100644 --- a/test/framework/easyconfigs/yeb/toy-0.0.yeb +++ b/test/framework/easyconfigs/yeb/toy-0.0.yeb @@ -23,7 +23,7 @@ checksums: [[ ]] patches: [ - 'toy-0.0_typo.patch', + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', ['toy-extra.txt', 'toy-0.0'], ] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d894b63a0a..8dbb098160 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -535,7 +535,8 @@ def test_read_write_file(self): def test_det_patched_files(self): """Test det_patched_files function.""" - pf = os.path.join(os.path.dirname(__file__), 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + pf = os.path.join(os.path.dirname(__file__), 'sandbox', 'sources', 'toy', toy_patch_fn) self.assertEqual(ft.det_patched_files(pf), ['b/toy-0.0/toy.source']) self.assertEqual(ft.det_patched_files(pf, omit_ab_prefix=True), ['toy-0.0/toy.source']) @@ -1083,7 +1084,8 @@ def test_is_patch_file(self): """Test for is_patch_file() function.""" testdir = os.path.dirname(os.path.abspath(__file__)) self.assertFalse(ft.is_patch_file(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))) - self.assertTrue(ft.is_patch_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch'))) + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + self.assertTrue(ft.is_patch_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn))) def test_is_alt_pypi_url(self): """Test is_alt_pypi_url() function.""" @@ -1133,7 +1135,8 @@ def test_apply_patch(self): testdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = self.test_prefix path = ft.extract_file(os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz'), tmpdir) - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) self.assertTrue(ft.apply_patch(toy_patch, path)) patched = ft.read_file(os.path.join(path, 'toy-0.0', 'toy.source')) @@ -1251,13 +1254,14 @@ def test_copy(self): testdir = os.path.dirname(os.path.abspath(__file__)) toy_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(testdir, 'sandbox', 'sources', 'toy', toy_patch_fn) gcc_dir = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'GCC') ft.copy([toy_file, gcc_dir, toy_patch], self.test_prefix) self.assertTrue(os.path.isdir(os.path.join(self.test_prefix, 'GCC'))) - for filepath in ['GCC/GCC-4.6.3.eb', 'GCC/GCC-4.9.2.eb', 'toy-0.0.eb', 'toy-0.0_typo.patch']: + for filepath in ['GCC/GCC-4.6.3.eb', 'GCC/GCC-4.9.2.eb', 'toy-0.0.eb', toy_patch_fn]: self.assertTrue(os.path.isfile(os.path.join(self.test_prefix, filepath))) # test copying of a single file, to a non-existing directory diff --git a/test/framework/github.py b/test/framework/github.py index 89fd2ba5f5..541e8551dd 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -270,7 +270,7 @@ def test_find_patches(self): 'validate': False, }) self.mock_stdout(True) - ec = gh.find_software_name_for_patch('toy-0.0_typo.patch') + ec = gh.find_software_name_for_patch('toy-0.0_fix-silly-typo-in-printf-statement.patch') txt = self.get_stdout() self.mock_stdout(False) diff --git a/test/framework/options.py b/test/framework/options.py index ac8c80a323..ad5dd7a36c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2441,7 +2441,8 @@ def test_new_update_pr(self): topdir = os.path.dirname(os.path.abspath(__file__)) test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') toy_ec = os.path.join(self.test_prefix, 'toy.eb') - toy_patch = os.path.join(topdir, 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(topdir, 'sandbox', 'sources', 'toy', toy_patch_fn) # purposely picked one with non-default toolchain/versionsuffix copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'), toy_ec) @@ -2515,7 +2516,7 @@ def test_new_update_pr(self): regexs[-1] = r"^\s*2 files changed" regexs.remove(r"^\* title: \"\{tools\}\[gompi/1.3.12\] toy v0.0\"") regexs.append(r"^\* title: \"just a test\"") - regexs.append(r".*/toy-0.0_typo.patch\s*\|") + regexs.append(r".*_fix-silly-typo-in-printf-statement.patch\s*\|") for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -2546,7 +2547,7 @@ def test_new_update_pr(self): r"^\* title: \"test-1-2-3\"", r"^\* overview of changes:", r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", - r".*/toy-0.0_typo.patch\s*\|", + r".*_fix-silly-typo-in-printf-statement.patch\s*\|", r"^\s*2 files changed", ] for regex in regexs: @@ -3385,16 +3386,19 @@ def test_inject_checksums(self): toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5' + bar_tar_gz_sha256 = 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7' + bar_patch = 'bar-0.0_fix-silly-typo-in-printf-statement.patch' + bar_patch_sha256 = '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab' patterns = [ "^== injecting sha256 checksums in .*/test\.eb$", "^== fetching sources & patches for test\.eb\.\.\.$", "^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+\.\.\.$", "^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$", "^== \* toy-0.0\.tar\.gz: %s$" % toy_source_sha256, - "^== \* toy-0\.0_typo\.patch: %s$" % toy_patch_sha256, + "^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: %s$" % toy_patch_sha256, "^== injecting sha256 checksums for extensions in test\.eb\.\.\.$", - "^== \* bar-0\.0\.tar\.gz: f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7$", - "^== \* bar-0\.0_typo\.patch: 84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab$", + "^== \* bar-0\.0\.tar\.gz: %s$" % bar_tar_gz_sha256, + "^== \* %s: %s$" % (bar_patch, bar_patch_sha256), "^== \* barbar-0\.0\.tar\.gz: a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c$", ] for pattern in patterns: @@ -3409,6 +3413,15 @@ def test_inject_checksums(self): # some checks on 'raw' easyconfig contents # single-line checksum for barbar extension since there's only one self.assertTrue("'checksums': ['a33100d1837d6d54edff7d19f195056c4bd9a4c8d399e72feaf90f0216c4c91c']," in ec_txt) + + # single-line checksum entry for bar source tarball + regex = re.compile("^[ ]*'%s', # bar-0.0.tar.gz$" % bar_tar_gz_sha256, re.M) + self.assertTrue(regex.search(ec_txt), "Pattern '%s' found in: %s" % (regex.pattern, ec_txt)) + + # no single-line checksum entry for bar*.patch, since line would be > 120 chars + regex = re.compile("^[ ]*# %s\n[ ]*'%s',$" % (bar_patch, bar_patch_sha256), re.M) + self.assertTrue(regex.search(ec_txt), "Pattern '%s' found in: %s" % (regex.pattern, ec_txt)) + # name/version of toy should NOT be hardcoded in exts_list, 'name'/'version' parameters should be used self.assertTrue(' (name, version, {' in ec_txt) @@ -3424,17 +3437,17 @@ def test_inject_checksums(self): # no parse errors for updated easyconfig file... ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['sources'], ['%(name)s-%(version)s.tar.gz']) - self.assertEqual(ec['patches'], ['toy-0.0_typo.patch']) + self.assertEqual(ec['patches'], ['toy-0.0_fix-silly-typo-in-printf-statement.patch']) self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']}) self.assertEqual(ec['exts_list'][0], ('bar', '0.0', { 'buildopts': " && gcc bar.c -o anotherbar", 'checksums': [ - 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', - '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab', + bar_tar_gz_sha256, + bar_patch_sha256, ], 'exts_filter': ("cat | grep '^bar$'", '%(name)s'), - 'patches': ['bar-0.0_typo.patch'], + 'patches': [bar_patch], 'toy_ext_param': "mv anotherbar bar_bis", 'unknowneasyconfigparameterthatshouldbeignored': 'foo', })) @@ -3499,7 +3512,7 @@ def test_inject_checksums(self): "^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+\.\.\.$", "^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$", "^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$", - "^== \* toy-0\.0_typo\.patch: e6785e1a721fc8bf79892e3ef41557c0$", + "^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: e6785e1a721fc8bf79892e3ef41557c0$", "^== \* toy-extra\.txt: 3b0787b3bf36603ae1398c4a49097893$", ] for pattern in patterns: diff --git a/test/framework/sandbox/sources/toy/extensions/bar-0.0_typo.patch b/test/framework/sandbox/sources/toy/extensions/bar-0.0_fix-silly-typo-in-printf-statement.patch similarity index 100% rename from test/framework/sandbox/sources/toy/extensions/bar-0.0_typo.patch rename to test/framework/sandbox/sources/toy/extensions/bar-0.0_fix-silly-typo-in-printf-statement.patch diff --git a/test/framework/sandbox/sources/toy/toy-0.0_typo.patch b/test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch similarity index 100% rename from test/framework/sandbox/sources/toy/toy-0.0_typo.patch rename to test/framework/sandbox/sources/toy/toy-0.0_fix-silly-typo-in-printf-statement.patch diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index fe3d442bb5..d1e081973e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1026,10 +1026,10 @@ def test_toy_patches(self): installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') - patch_file = os.path.join(installdir, 'easybuild', 'toy-0.0_typo.patch') + patch_file = os.path.join(installdir, 'easybuild', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertTrue(os.path.exists(patch_file)) - archived_patch_file = os.path.join(repositorypath, 'toy', 'toy-0.0_typo.patch') + archived_patch_file = os.path.join(repositorypath, 'toy', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertTrue(os.path.isfile(archived_patch_file)) def test_toy_module_fulltxt(self): @@ -1759,7 +1759,7 @@ def test_toy_build_trace(self): patterns = [ "^ >> installation prefix: .*/software/toy/0\.0$", "^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", - "^ >> applying patch toy-0\.0_typo\.patch$", + "^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", "^ >> running command:\n\t\[started at: .*\]\n\t\[output logged in .*\]\n\tgcc toy.c -o toy\n" + " >> command completed: exit 0, ran in .*", '^' + '\n'.join([ From 42f19a463887ba5689c1226938fc418d10d3da9c Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 12 Mar 2018 23:34:34 +0100 Subject: [PATCH 161/256] add function make_checksum_lines to inject_checksums --- easybuild/framework/easyblock.py | 36 ++++++++++++++------------------ test/framework/easyblock.py | 4 ++-- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4d0314d60e..a87aa44874 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2916,12 +2916,25 @@ def inject_checksums(ecs, checksum_type): :param ecs: list of EasyConfig instances to inject checksums into corresponding files :param checksum_type: type of checksum to use """ + def make_checksum_lines(checksums, indent_level): + line_indent = INDENT_4SPACES * indent_level + checksum_lines = [] + for fn, checksum in checksums: + checksum_line = "%s'%s', # %s" % (line_indent, checksum, fn) + if len(checksum_line) > MAX_LINE_LENGTH: + checksum_lines.extend([ + "%s# %s" % (line_indent, fn), + "%s'%s'," % (line_indent, checksum), + ]) + else: + checksum_lines.append(checksum_line) + return checksum_lines + for ec in ecs: ec_fn = os.path.basename(ec['spec']) ectxt = read_file(ec['spec']) print_msg("injecting %s checksums in %s" % (checksum_type, ec['spec']), log=_log) - # get easyblock instance and make sure all sources/patches are available by running fetch_step print_msg("fetching sources & patches for %s..." % ec_fn, log=_log) app = get_easyblock_instance(ec) @@ -2954,15 +2967,7 @@ def inject_checksums(ecs, checksum_type): checksum_lines = ["checksums = ['%s']\n" % checksums[0][1]] else: checksum_lines = ['checksums = ['] - for fn, checksum in checksums: - check_line = "%s'%s', # %s" % (INDENT_4SPACES, checksum, fn) - if len(check_line) > MAX_LINE_LENGTH: - checksum_lines.extend([ - "%s# %s" % (INDENT_4SPACES, fn), - "%s'%s'," % (INDENT_4SPACES, checksum), - ]) - else: - checksum_lines.append(check_line) + checksum_lines.extend(make_checksum_lines(checksums, indent_level=1)) checksum_lines.append(']\n') checksums_txt = '\n'.join(checksum_lines) @@ -3052,16 +3057,7 @@ def inject_checksums(ecs, checksum_type): exts_list_lines.append("%s'checksums': ['%s']," % (INDENT_4SPACES * 2, checksum)) else: exts_list_lines.append("%s'checksums': [" % (INDENT_4SPACES * 2)) - for fn, checksum in ext_checksums: - line_indent = INDENT_4SPACES * 3 - ext_checksum_line = "%s'%s', # %s" % (line_indent, checksum, fn) - if len(ext_checksum_line) > MAX_LINE_LENGTH: - exts_list_lines.extend([ - "%s# %s" % (line_indent, fn), - "%s'%s'," % (line_indent, checksum), - ]) - else: - exts_list_lines.append(ext_checksum_line) + exts_list_lines.extend(make_checksum_lines(ext_checksums, indent_level=3)) exts_list_lines.append("%s]," % (INDENT_4SPACES * 2)) if ext_options or ext_checksums: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e7f80961c8..166c0b98c6 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -845,15 +845,15 @@ def test_fetch_patches(self): ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) + toy_patch = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' eb.fetch_patches() self.assertEqual(len(eb.patches), 2) - self.assertEqual(eb.patches[0]['name'], 'toy-0.0_fix-silly-typo-in-printf-statement.patch') + self.assertEqual(eb.patches[0]['name'], toy_patch) self.assertFalse('level' in eb.patches[0]) # reset eb.patches = [] - toy_patch = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' patches = [ (toy_patch, 0), # should also be level 0 (not None or something else) (toy_patch, 4), # should be level 4 From b6e22586d9e0b3af566cc9416264f68bf6960506 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 13 Mar 2018 09:23:39 +0100 Subject: [PATCH 162/256] allow for trailing whitespace in 'class' line in is_software_specific_easyblock function --- easybuild/tools/include.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index c69bff78a7..f7533c9cca 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -145,7 +145,7 @@ def verify_imports(pymods, pypkg, from_path): def is_software_specific_easyblock(module): """Determine whether Python module at specified location is a software-specific easyblock.""" - return bool(re.search('^class EB_.*\(.*\):$', read_file(module), re.M)) + return bool(re.search('^class EB_.*\(.*\):\s*$', read_file(module), re.M)) def include_easyblocks(tmpdir, paths): From 093f38e6f3048ed28a8483e804845dcd2affca03 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 13 Mar 2018 10:27:11 +0100 Subject: [PATCH 163/256] make sure GitHub token is used in test for --preview-pr --- test/framework/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/options.py b/test/framework/options.py index 2de5be616f..4a6b6e9b92 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2217,6 +2217,7 @@ def test_preview_pr(self): eb_file = os.path.join(test_ecs_path, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb') args = [ '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, '--preview-pr', eb_file, ] From 718b30652a011c7c740399a09f2b551a1af532ad Mon Sep 17 00:00:00 2001 From: oliviermattelaer <33414646+oliviermattelaer@users.noreply.github.com> Date: Tue, 13 Mar 2018 16:03:40 +0100 Subject: [PATCH 164/256] Update filetools.py when trying to download file with urllib2 adding to the header the field Accept since the absence of such field can lead to some security module to raise a 403 response --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5176ec7180..943b263f8f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -459,7 +459,7 @@ def download_file(filename, url, path, forced=False): attempt_cnt = 0 # use custom HTTP header - url_req = urllib2.Request(url, headers={'User-Agent': 'EasyBuild'}) + url_req = urllib2.Request(url, headers={'User-Agent': 'EasyBuild', "Accept" : "text/html"}) while not downloaded and attempt_cnt < max_attempts: try: From d67fdd3f65abfc0c33c8a0dd604b887dd53b4ac8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 14 Mar 2018 17:26:22 +0800 Subject: [PATCH 165/256] loosen commit message requirements w.r.t. to patches as long as all easyconfigs are new --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index a0bf60a6a5..40a1d3b34b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -689,9 +689,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if commit_msg: cnt = len(file_info['paths_in_repo']) _log.debug("Using specified commit message for all %d new/modified easyconfigs at once: %s", cnt, commit_msg) - elif all(file_info['new']) and not paths['patch_files'] and not paths['files_to_delete']: + elif all(file_info['new']) and not paths['files_to_delete']: # automagically derive meaningful commit message if all easyconfig files are new - commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) + commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']+paths['patch_files']) else: raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " "modifying/deleting easyconfigs and/or specifying patches") From 5ef6fc324c6d38c8790d79eefe51149a3ee00928 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 14 Mar 2018 17:59:06 +0800 Subject: [PATCH 166/256] properly add patches to automatic commit message --- easybuild/tools/github.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 40a1d3b34b..94b303f0ce 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -691,7 +691,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ _log.debug("Using specified commit message for all %d new/modified easyconfigs at once: %s", cnt, commit_msg) elif all(file_info['new']) and not paths['files_to_delete']: # automagically derive meaningful commit message if all easyconfig files are new - commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']+paths['patch_files']) + commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) + if paths['patch_files']: + commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths['patch_files']) else: raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " "modifying/deleting easyconfigs and/or specifying patches") From edd65a81ece6cbec713d755dc6c2938bea361f85 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 14 Mar 2018 18:08:58 +0800 Subject: [PATCH 167/256] ammend error message --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 94b303f0ce..51e33efe8c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -696,7 +696,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths['patch_files']) else: raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " - "modifying/deleting easyconfigs and/or specifying patches") + "modifying/deleting easyconfigs") # figure out to which software name patches relate, and copy them to the right place if paths['patch_files']: From db48eedb159faee524faa47e5534ff2fa550be40 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 14 Mar 2018 19:19:52 +0100 Subject: [PATCH 168/256] Handle hidden toolchain in det_toolchain_element_details. Need to look at both dependencies and hidden dependencies when going through toolchains. --- easybuild/tools/module_naming_scheme/toolchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 4d6e686cf7..37ef3543e3 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -67,8 +67,9 @@ def det_toolchain_element_details(tc, elem): _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] tc_deps = tc_ec['dependencies'] + tc_hiddendeps = tc_ec['hiddendependencies'] tc_elem_details = None - for tc_dep in tc_deps: + for tc_dep in tc_deps + tc_hiddendeps: if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) From 6d3dd3f3a463c928b26938ebcc072fa76bc306de Mon Sep 17 00:00:00 2001 From: Riccardo Murri Date: Wed, 14 Mar 2018 20:17:01 +0100 Subject: [PATCH 169/256] Make `--job-max-jobs` default to "no limits". --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9e12020271..02b8874c14 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -634,7 +634,7 @@ def job_options(self): opts = OrderedDict({ 'backend-config': ("Configuration file for job backend", None, 'store', None), 'cores': ("Number of cores to request per job", 'int', 'store', None), - 'max-jobs': ("Maximum number of concurrent jobs (queued or running)", 'int', 'store', 50), + 'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), 'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0), From 149b6d2dd6dbe411467e74877bab53107aca2bb1 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 15 Mar 2018 09:51:25 +0800 Subject: [PATCH 170/256] make diff_stat pattern also match output of older git versions --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4a6b6e9b92..a3292e8519 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2491,7 +2491,7 @@ def test_new_update_pr(self): r"\(created using `eb --new-pr`\)", # description r"^\* overview of changes:", r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", - r"^\s*1 file changed", + r"^\s*1 file(s?) changed", ] for regex in regexs: regex = re.compile(regex, re.M) From f1043c70f72c599494869304eeacea731c8b951f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 15 Mar 2018 12:32:42 +0800 Subject: [PATCH 171/256] make diff_stat pattern also match output of older git versions (in two other places) --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index a3292e8519..9577f5090b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2608,7 +2608,7 @@ def test_new_update_pr(self): r"^== Determined branch name corresponding to easybuilders/easybuild-easyconfigs PR #2237: develop", r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", - r"^\s*1 file changed", + r"^\s*1 file(s?) changed", "^== pushing branch 'develop' to remote '.*' \(git@github.com:easybuilders/easybuild-easyconfigs.git\)", r"^Updated easybuilders/easybuild-easyconfigs PR #2237 by pushing to branch easybuilders/develop \[DRY RUN\]", ] @@ -2669,7 +2669,7 @@ def test_new_pr_delete(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", r'title: "delete bzip2-1.6.0"', - r"1 file changed, [0-9]+ deletions\(-\)", + r"1 file(s?) changed, [0-9]+ deletions\(-\)", ] for regex in regexs: regex = re.compile(regex, re.M) From 6e903786549bd46e37473345f6f50e6cd12bfae2 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 15 Mar 2018 12:54:58 +0800 Subject: [PATCH 172/256] test for custom commit message when deleting a file instead of when adding a patch following the change in PR #2438 --- test/framework/options.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4a6b6e9b92..e7c026a6a9 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2497,10 +2497,10 @@ def test_new_update_pr(self): regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) - # a custom commit message is required when doing more than just adding new easyconfigs (e.g., adding a patch) + # a custom commit message is required when doing more than just adding new easyconfigs (e.g., deleting a file) args.extend([ '--git-working-dirs-path=%s' % git_working_dir, - toy_patch, + ':bzip2-1.0.6.eb', ]) error_msg = "A meaningful commit message must be specified via --pr-commit-msg" @@ -2518,7 +2518,8 @@ def test_new_update_pr(self): regexs[-1] = r"^\s*2 files changed" regexs.remove(r"^\* title: \"\{tools\}\[gompi/1.3.12\] toy v0.0\"") regexs.append(r"^\* title: \"just a test\"") - regexs.append(r".*_fix-silly-typo-in-printf-statement.patch\s*\|") + regexs.append(r".*/bzip2-1.0.6.eb\s*\|") + regexs.append(r".*[0-9]+ deletions\(-\)") for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -2549,8 +2550,9 @@ def test_new_update_pr(self): r"^\* title: \"test-1-2-3\"", r"^\* overview of changes:", r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", - r".*_fix-silly-typo-in-printf-statement.patch\s*\|", + r".*/bzip2-1.0.6.eb\s*\|", r"^\s*2 files changed", + r".*[0-9]+ deletions\(-\)", ] for regex in regexs: regex = re.compile(regex, re.M) From 73d970c8d47a3bfa5508845eb97bee36b0845581 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 14 Mar 2018 16:47:22 +0800 Subject: [PATCH 173/256] add force_in_dry_run=True to copy_file in copy_patch_files --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b62a5c5430..8b067b508e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1579,7 +1579,7 @@ def copy_patch_files(patch_specs, target_dir): } for patch_path, soft_name in patch_specs: target_path = det_location_for(patch_path, target_dir, soft_name, os.path.basename(patch_path)) - copy_file(patch_path, target_path) + copy_file(patch_path, target_path, force_in_dry_run=True) patched_files['paths_in_repo'].append(target_path) return patched_files From 033cf942f847d288e8a928ba138fbc043fd2ff7a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 15 Mar 2018 17:21:55 +0000 Subject: [PATCH 174/256] Add -fno-math-errno to defaultprec/loose/veryloose GCC toolchainopts Neither PGI, nor Intel, nor Clang set errno for math functions by default. I have never seen it used. Setting this allows vectorization and inlining of sqrt in particular. --- easybuild/toolchains/compiler/gcc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index a494f96401..02e5aec394 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -62,9 +62,9 @@ class Gcc(Compiler): 'ieee': ['mieee-fp', 'fno-trapping-math'], 'strict': ['mieee-fp', 'mno-recip'], 'precise':['mno-recip'], - 'defaultprec':[], - 'loose': ['mrecip', 'mno-ieee-fp'], - 'veryloose': ['mrecip=all', 'mno-ieee-fp'], + 'defaultprec':['fno-math-errno'], + 'loose': ['fno-math-errno', 'mrecip', 'mno-ieee-fp'], + 'veryloose': ['fno-math-errno', 'mrecip=all', 'mno-ieee-fp'], 'defaultopt': ['O2', 'ftree-vectorize'], } From 04a0663608663380300a698c4c8c49e033db503c Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 15 Mar 2018 18:01:27 +0000 Subject: [PATCH 175/256] Adjust test cases to deal with -fno-math-errno. --- test/framework/options.py | 2 +- test/framework/toolchain.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 1751fddff3..efc93afd0a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2961,7 +2961,7 @@ def test_dump_env_config(self): "module load hwloc/1.6.2-GCC-4.7.2", # loading of dependency module # defining build env "export FC='gfortran'", - "export CFLAGS='-O2 -ftree-vectorize -march=native'", + "export CFLAGS='-O2 -ftree-vectorize -march=native -fno-math-errno'", ] for pattern in patterns: regex = re.compile("^%s$" % pattern, re.M) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 82d7d11954..fb53f349b0 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -476,20 +476,20 @@ def test_precision_flags(self): flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] - # check default precision: no specific flag for GCC + # check default precision: -fno-math-errno flag for GCC tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({}) tc.prepare() for var in flag_vars: - self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native") + self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native -fno-math-errno") # check other precision flags prec_flags = { 'ieee': "-mieee-fp -fno-trapping-math", 'strict': "-mieee-fp -mno-recip", 'precise': "-mno-recip", - 'loose': "-mrecip -mno-ieee-fp", - 'veryloose': "-mrecip=all -mno-ieee-fp", + 'loose': "-fno-math-errno -mrecip -mno-ieee-fp", + 'veryloose': "-fno-math-errno -mrecip=all -mno-ieee-fp", } for prec in prec_flags: for enable in [True, False]: @@ -500,7 +500,7 @@ def test_precision_flags(self): if enable: self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native %s" % prec_flags[prec]) else: - self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native") + self.assertEqual(os.getenv(var), "-O2 -ftree-vectorize -march=native -fno-math-errno") self.modtool.purge() def test_cgoolf_toolchain(self): @@ -580,7 +580,7 @@ def test_goolfc(self): tc.prepare() nvcc_flags = r' '.join([ - r'-Xcompiler="-O2 -ftree-vectorize -%s -fopenmp"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)], + r'-Xcompiler="-O2 -ftree-vectorize -%s -fno-math-errno -fopenmp"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)], # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), @@ -903,9 +903,9 @@ def test_independence(self): tc_cflags = { 'CrayCCE': "-O2 -homp -craype-verbose", - 'CrayGNU': "-O2 -fopenmp -craype-verbose", + 'CrayGNU': "-O2 -fno-math-errno -fopenmp -craype-verbose", 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose", - 'GCC': "-O2 -ftree-vectorize -test -fopenmp", + 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp", } From 6d2d2d1b94731b99c803153af90efa93c60753ab Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 15 Mar 2018 19:20:48 +0100 Subject: [PATCH 176/256] Just use tc_ec.dependencies() instead of explicitly using tc_ec['dependencies'] and tc_ec['hiddendependencies'] --- easybuild/tools/module_naming_scheme/toolchain.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 37ef3543e3..5d7f042448 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -66,10 +66,8 @@ def det_toolchain_element_details(tc, elem): _log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict) _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] - tc_deps = tc_ec['dependencies'] - tc_hiddendeps = tc_ec['hiddendependencies'] tc_elem_details = None - for tc_dep in tc_deps + tc_hiddendeps: + for tc_dep in tc_ec.dependencies(): if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) From c7216500f4e8ead31ef91fcf9e5f9a90201cfe95 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 16 Mar 2018 00:38:13 +0000 Subject: [PATCH 177/256] Fix one last precision testcase for ieee. --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index fb53f349b0..0bacbf7a87 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -485,7 +485,7 @@ def test_precision_flags(self): # check other precision flags prec_flags = { - 'ieee': "-mieee-fp -fno-trapping-math", + 'ieee': "-fno-math-errno -mieee-fp -fno-trapping-math", 'strict': "-mieee-fp -mno-recip", 'precise': "-mno-recip", 'loose': "-fno-math-errno -mrecip -mno-ieee-fp", From cfa35ee3ab8027edf395aba5d9780dc3a996e26f Mon Sep 17 00:00:00 2001 From: Davide Vanzo Date: Fri, 16 Mar 2018 09:48:31 -0500 Subject: [PATCH 178/256] Replaced raw string with bytes literal Useful for future Python 3 compatibility since raw strings do not seem to work as interator sentinel --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5176ec7180..dae20d23df 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -154,7 +154,7 @@ class ZlibChecksum(object): """ def __init__(self, algorithm): self.algorithm = algorithm - self.checksum = algorithm(r'') # use the same starting point as the module + self.checksum = algorithm(b'') # use the same starting point as the module self.blocksize = 64 # The same as md5/sha1 def update(self, data): @@ -653,7 +653,7 @@ def calc_block_checksum(path, algorithm): try: f = open(path, 'rb') - for block in iter(lambda: f.read(blocksize), r''): + for block in iter(lambda: f.read(blocksize), b''): algorithm.update(block) f.close() except IOError, err: From f1f20c09550ee6a01494a9d19caa16f383e22c7c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 21:32:32 +0100 Subject: [PATCH 179/256] include running of 'eb --check-github' in Travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1fb7ed6ec6..e9a1308740 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,6 +86,8 @@ script: # to run tests on an *installed* version of the EasyBuild framework; # this is done to catch possible packaging issues - cd $HOME + # check GitHub configuration + - eb --check-github # run test suite - python -O -m test.framework.suite # check bootstrap script version From 53d9ed9cb1651a69662caae7c78e9533939789ae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 21:42:59 +0100 Subject: [PATCH 180/256] add config file for houndci --- .hound.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .hound.yml diff --git a/.hound.yml b/.hound.yml new file mode 100644 index 0000000000..79c6e75fa0 --- /dev/null +++ b/.hound.yml @@ -0,0 +1,2 @@ +flake8: + enabled: true From 6b5b8b17d174e1f90639aa439c7dc701ba829633 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 21:48:25 +0100 Subject: [PATCH 181/256] add custom flake8 config and let houndci use it --- .flake8.ini | 2 ++ .hound.yml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 .flake8.ini diff --git a/.flake8.ini b/.flake8.ini new file mode 100644 index 0000000000..6deafc2617 --- /dev/null +++ b/.flake8.ini @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/.hound.yml b/.hound.yml index 79c6e75fa0..2916903576 100644 --- a/.hound.yml +++ b/.hound.yml @@ -1,2 +1,4 @@ +# configuration for houndci, see https://houndci.com/configuration#python flake8: enabled: true + config_file: .flake8.ini From 688adfedeab327279c104e618a4d62ffc2d8abd4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 21:53:12 +0100 Subject: [PATCH 182/256] fix trailing whitespace --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 51e33efe8c..08ab9245cc 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -692,7 +692,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ elif all(file_info['new']) and not paths['files_to_delete']: # automagically derive meaningful commit message if all easyconfig files are new commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) - if paths['patch_files']: + if paths['patch_files']: commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths['patch_files']) else: raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " From 2f338567b33012e4d583f022b6671d858c9c8a34 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 22:06:39 +0100 Subject: [PATCH 183/256] configure modules tool + syntax both for tests & EasyBuild itself --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index e9a1308740..2f30c5c19f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,6 +74,8 @@ script: fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi + - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi + - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi # create 'source distribution' tarball, like we do when publishing a release to PyPI - cd $TRAVIS_BUILD_DIR; python setup.py sdist - ls -l $TRAVIS_BUILD_DIR/dist/ From 8c0483ab31a4fe591f4559c2ad191ad763ba4cad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Mar 2018 21:54:33 +0100 Subject: [PATCH 184/256] fix multiple spaces before operator --- test/framework/options.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index a1344aa533..ada48bcdfa 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1929,7 +1929,7 @@ def test_xxx_include_easyblocks(self): test_easyblocks = os.path.dirname(os.path.abspath(__file__)) path_pattern = os.path.join(test_easyblocks, 'sandbox', 'easybuild', 'easyblocks', 'f', 'foo.py') - foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) # 'undo' import of foo easyblock @@ -1961,7 +1961,7 @@ def test_xxx_include_easyblocks(self): logtxt = read_file(self.logfile) path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', 'foo.py') - foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) # easyblock is found via get_easyblock_class @@ -2005,7 +2005,7 @@ def test_xxx_include_generic_easyblocks(self): path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', 'generic', 'foobar.py') - foo_regex = re.compile(r"^\|-- FooBar \(easybuild.easyblocks.generic.foobar @ %s\)" % path_pattern, re.M) + foo_regex = re.compile(r"^\|-- FooBar \(easybuild.easyblocks.generic.foobar @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) klass = get_easyblock_class('FooBar') @@ -2043,7 +2043,7 @@ def test_xxx_include_generic_easyblocks(self): mod_pattern = 'easybuild.easyblocks.generic.generictest' path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', 'generic', 'generictest.py') - foo_regex = re.compile(r"^\|-- GenericTest \(%s @ %s\)" % (mod_pattern, path_pattern), re.M) + foo_regex = re.compile(r"^\|-- GenericTest \(%s @ %s\)" % (mod_pattern, path_pattern), re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) klass = get_easyblock_class('GenericTest') From 7e481922c2100f42a025bcd7976a92a3059032f1 Mon Sep 17 00:00:00 2001 From: Olivier Mattelaer Date: Sun, 18 Mar 2018 16:44:49 +0100 Subject: [PATCH 185/256] pass to */* as suggested --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 943b263f8f..dff43dc68c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -459,7 +459,7 @@ def download_file(filename, url, path, forced=False): attempt_cnt = 0 # use custom HTTP header - url_req = urllib2.Request(url, headers={'User-Agent': 'EasyBuild', "Accept" : "text/html"}) + url_req = urllib2.Request(url, headers={'User-Agent': 'EasyBuild', "Accept" : "*/*"}) while not downloaded and attempt_cnt < max_attempts: try: From a7f490e5470e4c27577f7554c7423fa2737ca073 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 16 Mar 2018 17:46:07 +0800 Subject: [PATCH 186/256] flesh out common code block for asserting list of regexs --- test/framework/options.py | 40 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index a1344aa533..381e811a0b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2433,6 +2433,14 @@ def test_fixed_installdir_naming_scheme(self): app.gen_installdir() self.assertTrue(app.installdir.endswith('software/toy/0.0')) + def _assert_regexs(self, regexs, txt, assert_true=True): + for regex in regexs: + regex = re.compile(regex, re.M) + if assert_true: + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + else: + self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) + def test_new_update_pr(self): """Test use of --new-pr (dry run only).""" if self.github_token is None: @@ -2493,9 +2501,7 @@ def test_new_update_pr(self): r".*/toy-0.0-gompi-1.3.12-test.eb\s*\|", r"^\s*1 file(s?) changed", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) # a custom commit message is required when doing more than just adding new easyconfigs (e.g., deleting a file) args.extend([ @@ -2520,9 +2526,7 @@ def test_new_update_pr(self): regexs.append(r"^\* title: \"just a test\"") regexs.append(r".*/bzip2-1.0.6.eb\s*\|") regexs.append(r".*[0-9]+ deletions\(-\)") - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) GITHUB_TEST_ORG = 'test-organization' args.extend([ @@ -2554,9 +2558,7 @@ def test_new_update_pr(self): r"^\s*2 files changed", r".*[0-9]+ deletions\(-\)", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) # modifying an existing easyconfig requires a custom PR title gcc_ec = os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb') @@ -2614,9 +2616,7 @@ def test_new_update_pr(self): "^== pushing branch 'develop' to remote '.*' \(git@github.com:easybuilders/easybuild-easyconfigs.git\)", r"^Updated easybuilders/easybuild-easyconfigs PR #2237 by pushing to branch easybuilders/develop \[DRY RUN\]", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) # also check behaviour under --extended-dry-run/-x args.remove('-D') @@ -2632,9 +2632,7 @@ def test_new_update_pr(self): r"^\+\+\+\s*.*toy-0.0-gompi-1.3.12-test.eb", r"^\+name = 'toy'", ]) - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) # check whether comments/buildstats get filtered out regexs = [ @@ -2642,9 +2640,7 @@ def test_new_update_pr(self): "# Build statistics", "buildstats\s*=", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt, assert_true=False) def test_new_pr_delete(self): """Test use of --new-pr to delete easyconfigs.""" @@ -2673,9 +2669,7 @@ def test_new_pr_delete(self): r'title: "delete bzip2-1.6.0"', r"1 file(s?) changed, [0-9]+ deletions\(-\)", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) def test_new_pr_dependencies(self): """Test use of --new-pr with automatic dependency lookup.""" @@ -2726,9 +2720,7 @@ def test_new_pr_dependencies(self): r"^\s*2 files changed", ] - for regex in regexs: - regex = re.compile(regex, re.M) - self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + self._assert_regexs(regexs, txt) def test_merge_pr(self): """ From f7e1a766f4ec9726e15052fefc2efe82b1436ea7 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 29 Jan 2018 14:37:44 +0800 Subject: [PATCH 187/256] automatically add 'new' and/or 'update' labels in --new-pr --- easybuild/framework/easyconfig/easyconfig.py | 7 +++++++ easybuild/tools/github.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b62a5c5430..280bbde87f 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1541,6 +1541,8 @@ def copy_easyconfigs(paths, target_dir): 'ecs': [], 'paths_in_repo': [], 'new': [], + 'new_folder': [], + 'new_file_in_existing_folder': [], } for path in paths: @@ -1554,6 +1556,11 @@ def copy_easyconfigs(paths, target_dir): target_path = det_location_for(path, target_dir, soft_name, ec_filename) file_info['new'].append(not os.path.exists(target_path)) + + file_info['new_folder'].append(not os.path.exists(os.path.dirname(target_path))) + + file_info['new_file_in_existing_folder'].append(file_info['new'][-1] and not file_info['new_folder'][-1]) + copy_file(path, target_path, force_in_dry_run=True) file_info['paths_in_repo'].append(target_path) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 51e33efe8c..e3379fc2ec 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1073,6 +1073,13 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): start_account=pr_target_account, commit_msg=commit_msg) + # label easyconfigs for new software and/or new easyconfigs for existing software + labels = [] + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + # only use most common toolchain(s) in toolchain label of PR title toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) @@ -1112,6 +1119,7 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): "* target: %s/%s:%s" % (pr_target_account, pr_target_repo, pr_target_branch), "* from: %s/%s:%s" % (github_account, pr_target_repo, branch), "* title: \"%s\"" % title, + "* labels: %s" % labels, "* description:", '"""', full_descr, @@ -1136,6 +1144,15 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False) + # post labels + pr = data['html_url'].split('/')[-1] + pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] + status, data = pr_url.labels.post(body=labels) + if not status == HTTP_STATUS_OK: + raise EasyBuildError("Failed to add labels to PR# %s; status %s, data: %s", pr, status, data) + + print_msg("Added labels %s to PR#%s" % (labels, pr), log=_log, prefix=False) + @only_if_module_is_available('git', pkgname='GitPython') def update_pr(pr, paths, ecs, commit_msg=None): From 06d475033a8ae81df2b285af17caf6adf090295a Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 19 Mar 2018 15:00:39 +0800 Subject: [PATCH 188/256] cleanup code and guard block with if labels --- easybuild/framework/easyconfig/easyconfig.py | 10 +++++----- easybuild/tools/github.py | 17 +++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 280bbde87f..f10fc63bd6 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1555,11 +1555,11 @@ def copy_easyconfigs(paths, target_dir): target_path = det_location_for(path, target_dir, soft_name, ec_filename) - file_info['new'].append(not os.path.exists(target_path)) - - file_info['new_folder'].append(not os.path.exists(os.path.dirname(target_path))) - - file_info['new_file_in_existing_folder'].append(file_info['new'][-1] and not file_info['new_folder'][-1]) + new_file = not os.path.exists(target_path) + new_folder = not os.path.exists(os.path.dirname(target_path)) + file_info['new'].append(new_file) + file_info['new_folder'].append(new_folder) + file_info['new_file_in_existing_folder'].append(new_file and not new_folder) copy_file(path, target_path, force_in_dry_run=True) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e3379fc2ec..4a518b9728 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1119,7 +1119,7 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): "* target: %s/%s:%s" % (pr_target_account, pr_target_repo, pr_target_branch), "* from: %s/%s:%s" % (github_account, pr_target_repo, branch), "* title: \"%s\"" % title, - "* labels: %s" % labels, + "* labels: %s" % (', '.join(labels) or '(none)'), "* description:", '"""', full_descr, @@ -1144,14 +1144,15 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False) - # post labels - pr = data['html_url'].split('/')[-1] - pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] - status, data = pr_url.labels.post(body=labels) - if not status == HTTP_STATUS_OK: - raise EasyBuildError("Failed to add labels to PR# %s; status %s, data: %s", pr, status, data) + if labels: + # post labels + pr = data['html_url'].split('/')[-1] + pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] + status, data = pr_url.labels.post(body=labels) + if not status == HTTP_STATUS_OK: + raise EasyBuildError("Failed to add labels to PR# %s; status %s, data: %s", pr, status, data) - print_msg("Added labels %s to PR#%s" % (labels, pr), log=_log, prefix=False) + print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) @only_if_module_is_available('git', pkgname='GitPython') From d84cbee7989daf999ddc8480a032f70a0ee711f1 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 19 Mar 2018 16:01:03 +0800 Subject: [PATCH 189/256] update file_info fields in test_copy_easyconfig --- test/framework/easyconfig.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 61a787c2bb..50ffdd4723 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1821,7 +1821,8 @@ def test_copy_easyconfigs(self): # passing an empty list of paths is fine res = copy_easyconfigs([], target_dir) - self.assertEqual(res, {'ecs': [], 'new': [], 'paths_in_repo': []}) + self.assertEqual(res, {'ecs': [], 'new': [], 'new_file_in_existing_folder': [], + 'new_folder': [], 'paths_in_repo': []}) self.assertEqual(os.listdir(ecs_target_dir), []) # copy test easyconfigs, purposely under a different name @@ -1837,7 +1838,8 @@ def test_copy_easyconfigs(self): shutil.copy2(os.path.join(test_ecs_dir, src_ec), ecs_to_copy[-1]) res = copy_easyconfigs(ecs_to_copy, target_dir) - self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'paths_in_repo']) + self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'new_file_in_existing_folder', + 'new_folder', 'paths_in_repo']) self.assertEqual(len(res['ecs']), len(test_ecs)) self.assertTrue(all(isinstance(ec, EasyConfig) for ec in res['ecs'])) self.assertTrue(all(res['new'])) @@ -1869,7 +1871,7 @@ def test_copy_easyconfigs(self): # copy single easyconfig with buildstats included for running further tests res = copy_easyconfigs([toy_ec], target_dir) - self.assertEqual([len(x) for x in res.values()], [1, 1, 1]) + self.assertEqual([len(x) for x in res.values()], [1, 1, 1, 1, 1]) self.assertEqual(res['ecs'][0].full_mod_name, 'toy/0.0') # toy-0.0.eb was already copied into target_dir, so should not be marked as new anymore From f8ca3227428ddb5ed29eb1fc86a47363636682d1 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 19 Mar 2018 16:27:09 +0800 Subject: [PATCH 190/256] flesh out common code block for running mock eb --- test/framework/options.py | 176 ++++++++++---------------------------- 1 file changed, 43 insertions(+), 133 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 381e811a0b..c56e9c0fbf 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2441,6 +2441,19 @@ def _assert_regexs(self, regexs, txt, assert_true=True): else: self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) + def _run_mock_eb(self, args, stdout=True, stderr=False, do_build=False, raise_error=False, verbose=False, testing=False, strip=False): + self.mock_stdout(stdout) + self.mock_stderr(stderr) + self.eb_main(args, do_build=do_build, raise_error=raise_error, verbose=verbose, testing=testing) + stdout_txt = self.get_stdout() + stderr_txt = self.get_stderr() if stderr else "" + self.mock_stdout(False) + self.mock_stderr(False) + if strip: + stdout_txt = stdout_txt.strip() + stderr_txt = stderr_txt.strip() + return stdout_txt, stderr_txt + def test_new_update_pr(self): """Test use of --new-pr (dry run only).""" if self.github_token is None: @@ -2476,10 +2489,7 @@ def test_new_update_pr(self): '-D', '--disable-cleanup-tmpdir', ] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) # determine location of repo clone, can be used to test --git-working-dirs-path (and save time) dirs = glob.glob(os.path.join(self.test_prefix, 'eb-*', '*', 'git-working-dir*')) @@ -2516,10 +2526,7 @@ def test_new_update_pr(self): # add required commit message, try again args.append('--pr-commit-msg=just a test') - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs[-1] = r"^\s*2 files changed" regexs.remove(r"^\* title: \"\{tools\}\[gompi/1.3.12\] toy v0.0\"") @@ -2539,10 +2546,7 @@ def test_new_update_pr(self): '--pr-target-account=boegel', # we need to be able to 'clone' from here (via https) '--pr-title=test-1-2-3', ]) - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs = [ r"^== fetching branch 'master' from https://github.com/boegel/easybuild-easyconfigs.git...", @@ -2578,10 +2582,7 @@ def test_new_update_pr(self): # also specifying commit message is sufficient; PR title is inherited from commit message args.append('--pr-commit-msg=this is just a test') - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regex = re.compile('^\* title: "this is just a test"', re.M) self.assertTrue(regex.search(txt), "Pattern '%s' is found in: %s" % (regex.pattern, txt)) @@ -2603,10 +2604,7 @@ def test_new_update_pr(self): self.mock_stdout(False) args.append('--pr-commit-msg="just a test"') - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs = [ r"^== Determined branch name corresponding to easybuilders/easybuild-easyconfigs PR #2237: develop", @@ -2622,10 +2620,7 @@ def test_new_update_pr(self): args.remove('-D') args.append('-x') - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs.extend([ r"Full patch:", @@ -2658,11 +2653,7 @@ def test_new_pr_delete(self): '--pr-title=delete bzip2-1.6.0', '--pr-commit-msg="delete bzip2-1.6.0.eb"' ] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) - + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", @@ -2708,10 +2699,7 @@ def test_new_pr_dependencies(self): '-r%s' % self.test_prefix, ] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) regexs = [ r"^\* overview of changes:", @@ -2737,13 +2725,7 @@ def test_merge_pr(self): ] # merged PR for EasyBuild-3.3.0.eb, is missing approved review - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", @@ -2763,13 +2745,7 @@ def test_merge_pr(self): # full eligible merged PR args[1] = '4832' - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + stdour, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4832 for merging...", @@ -2796,13 +2772,7 @@ def test_merge_pr(self): '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, ] - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", @@ -2864,10 +2834,7 @@ def test_show_config(self): write_file(cfgfile, cfgtxt) args = ['--configfiles=%s' % cfgfile, '--show-config', '--buildpath=/weird/build/dir'] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout().strip() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) default_prefix = os.path.join(os.environ['HOME'], '.local', 'easybuild') @@ -2892,10 +2859,7 @@ def test_show_config(self): self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) args = ['--configfiles=%s' % cfgfile, '--show-full-config', '--buildpath=/weird/build/dir'] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) # output of --show-full-config includes additional lines for options with default values expected_lines.extend([ @@ -2912,10 +2876,7 @@ def test_show_config(self): # (existing config files are ignored via $EASYBUILD_IGNORECONFIGFILES) self.assertFalse(os.environ.get('EASYBUILD_CONFIGFILES', False)) args = ['--show-config', '--buildpath=/weird/build/dir'] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout().strip() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) self.assertTrue(re.search(r"buildpath\s* \(C\) = /weird/build/dir", txt)) # --show-config should not break including of easyblocks via $EASYBUILD_INCLUDE_EASYBLOCKS (see bug #1696) @@ -2930,10 +2891,7 @@ def test_show_config(self): os.environ['EASYBUILD_INCLUDE_EASYBLOCKS'] = testeasyblocktoinclude args = ['--show-config'] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout().strip() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) regex = re.compile(r'^include-easyblocks \(E\) = .*/testeasyblocktoinclude.py$', re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -2946,10 +2904,7 @@ def test_dump_env_config(self): args = ['%s.eb' % ec for ec in [fftw, gcc, openmpi]] + ['--dump-env-script'] os.chdir(self.test_prefix) - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout().strip() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) for name in [fftw, gcc, openmpi]: # check stdout @@ -2968,9 +2923,7 @@ def test_dump_env_config(self): os.chdir(self.test_prefix) args.append('--force') - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True) - self.mock_stdout(False) + self._run_mock_eb(args, args, do_build=True, raise_error=True) # check contents of script env_script = os.path.join(self.test_prefix, '%s.env' % openmpi) @@ -2997,10 +2950,7 @@ def test_dump_env_config(self): def test_stop(self): """Test use of --stop.""" args = ['toy-0.0.eb', '--force', '--stop=configure'] - self.mock_stdout(True) - self.eb_main(args, do_build=True, raise_error=True, testing=False) - txt = self.get_stdout().strip() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) regex = re.compile("COMPLETED: Installation STOPPED successfully", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -3115,10 +3065,8 @@ def test_list_software(self): '--list-software', '--robot-paths=%s' % test_ecs, ] - self.mock_stdout(True) - self.eb_main(args, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + expected = '\n'.join([ "== Processed 5/5 easyconfigs...", "== Found 2 different software packages", @@ -3134,10 +3082,7 @@ def test_list_software(self): '--output-format=rst', '--robot-paths=%s' % test_ecs, ] - self.mock_stdout(True) - self.eb_main(args, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args) self.assertTrue(re.search('^\*GCC\*', txt, re.M)) self.assertTrue(re.search('^``4.6.3``\s+``dummy``', txt, re.M)) self.assertTrue(re.search('^\*gzip\*', txt, re.M)) @@ -3148,10 +3093,7 @@ def test_list_software(self): '--output-format=rst', '--robot-paths=%s' % test_ecs, ] - self.mock_stdout(True) - self.eb_main(args, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args) self.assertTrue(re.search('== Processed 5/5 easyconfigs...', txt, re.M)) self.assertTrue(re.search('== Found 2 different software packages', txt, re.M)) self.assertTrue(re.search('== Retained 1 installed software packages', txt, re.M)) @@ -3162,10 +3104,7 @@ def test_list_software(self): '--list-installed-software=detailed', '--robot-paths=%s' % test_ecs, ] - self.mock_stdout(True) - self.eb_main(args, testing=False) - txt = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args) self.assertTrue(re.search('^== Retained 1 installed software packages', txt, re.M)) self.assertTrue(re.search('^\* GCC', txt, re.M)) self.assertTrue(re.search('^\s+\* GCC v4.6.3: dummy', txt, re.M)) @@ -3229,10 +3168,7 @@ def test_check_style(self): 'GCC-4.9.2.eb', 'toy-0.0.eb', ] - self.mock_stdout(True) - self.eb_main(args, raise_error=True) - stdout = self.get_stdout() - self.mock_stdout(False) + txt, _ = self._run_mock_eb(args, raise_error=True) regex = re.compile(r"Running style check on 2 easyconfig\(s\)", re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -3276,10 +3212,8 @@ def test_allow_use_as_root(self): self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, ['toy-0.0.eb'], raise_error=True) # running as root is allowed under --allow-use-as-root, but does result in a warning being printed to stderr - self.mock_stderr(True) - self.eb_main(['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences'], raise_error=True) - stderr = self.get_stderr().strip() - self.mock_stderr(False) + args = ['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences'] + _, stderr = self._run_mock_eb(args, stdout=False, stderr=True, raise_error=True, strip=True) expected = "WARNING: Using EasyBuild as root is NOT recommended, please proceed with care!\n" expected += "(this is only allowed because EasyBuild was configured with " @@ -3372,13 +3306,7 @@ def test_inject_checksums(self): self.assertEqual(stderr, '') args.append('--force') - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, raise_error=True) - stdout = self.get_stdout().strip() - stderr = self.get_stderr().strip() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5' @@ -3467,13 +3395,7 @@ def test_inject_checksums(self): ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, raise_error=True) - stdout = self.get_stdout().strip() - stderr = self.get_stderr().strip() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) @@ -3494,13 +3416,7 @@ def test_inject_checksums(self): write_file(test_ec, toy_ec_txt) args = [test_ec, '--inject-checksums=md5'] - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, raise_error=True) - stdout = self.get_stdout().strip() - stderr = self.get_stderr().strip() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) patterns = [ "^== injecting md5 checksums in .*/test\.eb$", @@ -3562,13 +3478,7 @@ def test_force_download(self): '--force-download', '--sourcepath=%s' % self.test_prefix, ] - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(args, raise_error=True, verbose=True, do_build=True) - stdout = self.get_stdout().strip() - stderr = self.get_stderr().strip() - self.mock_stdout(False) - self.mock_stderr(False) + stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True, verbose=True, strip=True) self.assertEqual(stdout, '') regex = re.compile("^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$") self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr)) From 79592ae36110b58fb2785dd5691651b820ba69a6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Mar 2018 09:53:56 +0100 Subject: [PATCH 191/256] ! -z -> -n --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f30c5c19f..7e9c344bcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,25 +57,25 @@ install: - easy_install vsc-base # install environment modules or Lmod - export INSTALL_DEP=$TRAVIS_BUILD_DIR/easybuild/scripts/install_eb_dep.sh - - if [ ! -z $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi - - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi - - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi - - if [ ! -z $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi + - if [ -n $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi + - if [ -n $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi + - if [ -n $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi + - if [ -n $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi script: # make sure 'ml' alias is defined, otherwise sourcing the init script fails (silently) for Lmod (< 5.9.3) - - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi + - if [ -n $MOD_INIT ] && [ -n $LMOD_VERSION ]; then alias ml=foobar; fi # set up environment for modules tool (if $MOD_INIT is defined) - - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi + - if [ -n $MOD_INIT ]; then source $MOD_INIT; type module; fi # install GitHub token - - if [ ! -z $GITHUB_TOKEN ]; then + - if [ -n $GITHUB_TOKEN ]; then if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi - - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi - - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi + - if [ -n $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi + - if [ -n $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi # create 'source distribution' tarball, like we do when publishing a release to PyPI - cd $TRAVIS_BUILD_DIR; python setup.py sdist - ls -l $TRAVIS_BUILD_DIR/dist/ From 30a4c5324d88a064ff7a36153ec843f52544882d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Mar 2018 10:10:12 +0100 Subject: [PATCH 192/256] -n -> ! -z --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e9c344bcf..2f30c5c19f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,25 +57,25 @@ install: - easy_install vsc-base # install environment modules or Lmod - export INSTALL_DEP=$TRAVIS_BUILD_DIR/easybuild/scripts/install_eb_dep.sh - - if [ -n $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi - - if [ -n $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi - - if [ -n $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi - - if [ -n $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi + - if [ ! -z $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi + - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi + - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi + - if [ ! -z $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi script: # make sure 'ml' alias is defined, otherwise sourcing the init script fails (silently) for Lmod (< 5.9.3) - - if [ -n $MOD_INIT ] && [ -n $LMOD_VERSION ]; then alias ml=foobar; fi + - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi # set up environment for modules tool (if $MOD_INIT is defined) - - if [ -n $MOD_INIT ]; then source $MOD_INIT; type module; fi + - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi # install GitHub token - - if [ -n $GITHUB_TOKEN ]; then + - if [ ! -z $GITHUB_TOKEN ]; then if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi - - if [ -n $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi - - if [ -n $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi + - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi + - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi # create 'source distribution' tarball, like we do when publishing a release to PyPI - cd $TRAVIS_BUILD_DIR; python setup.py sdist - ls -l $TRAVIS_BUILD_DIR/dist/ From cab1d2986587a83b9b82d0a5794d47ba6b7ea695 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 19 Mar 2018 17:26:53 +0800 Subject: [PATCH 193/256] also test for adding a patch in test_new_update_pr --- test/framework/options.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index a1344aa533..bb14a11a7c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2558,6 +2558,19 @@ def test_new_update_pr(self): regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + # should also work with a patch + args.append(toy_patch) + self.mock_stdout(True) + self.eb_main(args, do_build=True, raise_error=True, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) + + regexs[-2] = r"^\s*3 files changed" + regexs.append(r".*_fix-silly-typo-in-printf-statement.patch\s*\|") + for regex in regexs: + regex = re.compile(regex, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + # modifying an existing easyconfig requires a custom PR title gcc_ec = os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb') self.assertTrue(os.path.exists(gcc_ec)) From 573bad82bf42dca4317e94f42c77ea8b5b590662 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 09:47:12 +0800 Subject: [PATCH 194/256] add docstrings and simplify _run_mock_eb --- test/framework/options.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index c56e9c0fbf..fd373a7599 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2434,6 +2434,7 @@ def test_fixed_installdir_naming_scheme(self): self.assertTrue(app.installdir.endswith('software/toy/0.0')) def _assert_regexs(self, regexs, txt, assert_true=True): + """Helper function to assert presence/absence of list of regex patterns in a text""" for regex in regexs: regex = re.compile(regex, re.M) if assert_true: @@ -2441,12 +2442,13 @@ def _assert_regexs(self, regexs, txt, assert_true=True): else: self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) - def _run_mock_eb(self, args, stdout=True, stderr=False, do_build=False, raise_error=False, verbose=False, testing=False, strip=False): - self.mock_stdout(stdout) - self.mock_stderr(stderr) + def _run_mock_eb(self, args, do_build=False, raise_error=False, verbose=False, testing=False, strip=False): + """Helper function to mock easybuild runs""" + self.mock_stdout(True) + self.mock_stderr(True) self.eb_main(args, do_build=do_build, raise_error=raise_error, verbose=verbose, testing=testing) stdout_txt = self.get_stdout() - stderr_txt = self.get_stderr() if stderr else "" + stderr_txt = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) if strip: @@ -2725,7 +2727,7 @@ def test_merge_pr(self): ] # merged PR for EasyBuild-3.3.0.eb, is missing approved review - stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", @@ -2745,7 +2747,7 @@ def test_merge_pr(self): # full eligible merged PR args[1] = '4832' - stdour, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) + stdour, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4832 for merging...", @@ -2772,7 +2774,7 @@ def test_merge_pr(self): '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, ] - stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", @@ -3213,7 +3215,7 @@ def test_allow_use_as_root(self): # running as root is allowed under --allow-use-as-root, but does result in a warning being printed to stderr args = ['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences'] - _, stderr = self._run_mock_eb(args, stdout=False, stderr=True, raise_error=True, strip=True) + _, stderr = self._run_mock_eb(args, raise_error=True, strip=True) expected = "WARNING: Using EasyBuild as root is NOT recommended, please proceed with care!\n" expected += "(this is only allowed because EasyBuild was configured with " @@ -3306,7 +3308,7 @@ def test_inject_checksums(self): self.assertEqual(stderr, '') args.append('--force') - stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5' @@ -3395,7 +3397,7 @@ def test_inject_checksums(self): ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) - stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) @@ -3416,7 +3418,7 @@ def test_inject_checksums(self): write_file(test_ec, toy_ec_txt) args = [test_ec, '--inject-checksums=md5'] - stdout, stderr = self._run_mock_eb(args, stderr=True, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) patterns = [ "^== injecting md5 checksums in .*/test\.eb$", @@ -3478,7 +3480,7 @@ def test_force_download(self): '--force-download', '--sourcepath=%s' % self.test_prefix, ] - stdout, stderr = self._run_mock_eb(args, stderr=True, do_build=True, raise_error=True, verbose=True, strip=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True) self.assertEqual(stdout, '') regex = re.compile("^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$") self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr)) From 041a5964045ca115715e9f6f22fc1ff4da833382 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 13:38:48 +0800 Subject: [PATCH 195/256] make diff_stat pattern also match output of older git versions --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index fd373a7599..c91d9f584e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2660,7 +2660,7 @@ def test_new_pr_delete(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", r'title: "delete bzip2-1.6.0"', - r"1 file(s?) changed, [0-9]+ deletions\(-\)", + r"1 file(s?) changed,( 0 insertions\(\+\),)? [0-9]+ deletions\(-\)", ] self._assert_regexs(regexs, txt) From f7f360e61a041d5349f57f66bcccc31adbd32326 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 13:39:14 +0800 Subject: [PATCH 196/256] fix typos --- test/framework/options.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index c91d9f584e..119aa4da81 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2747,7 +2747,7 @@ def test_merge_pr(self): # full eligible merged PR args[1] = '4832' - stdour, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4832 for merging...", @@ -2925,7 +2925,7 @@ def test_dump_env_config(self): os.chdir(self.test_prefix) args.append('--force') - self._run_mock_eb(args, args, do_build=True, raise_error=True) + self._run_mock_eb(args, do_build=True, raise_error=True) # check contents of script env_script = os.path.join(self.test_prefix, '%s.env' % openmpi) @@ -3068,7 +3068,6 @@ def test_list_software(self): '--robot-paths=%s' % test_ecs, ] txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) - expected = '\n'.join([ "== Processed 5/5 easyconfigs...", "== Found 2 different software packages", @@ -3170,7 +3169,7 @@ def test_check_style(self): 'GCC-4.9.2.eb', 'toy-0.0.eb', ] - txt, _ = self._run_mock_eb(args, raise_error=True) + stdout, _ = self._run_mock_eb(args, raise_error=True) regex = re.compile(r"Running style check on 2 easyconfig\(s\)", re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) From 7762407876e2bd11f79108c102e84483075a1d57 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 18:14:42 +0800 Subject: [PATCH 197/256] pass testing=True to _run_mock_eb where needed --- test/framework/options.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 119aa4da81..16a458f1f9 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2925,7 +2925,7 @@ def test_dump_env_config(self): os.chdir(self.test_prefix) args.append('--force') - self._run_mock_eb(args, do_build=True, raise_error=True) + self._run_mock_eb(args, do_build=True, raise_error=True, testing=True) # check contents of script env_script = os.path.join(self.test_prefix, '%s.env' % openmpi) @@ -3169,7 +3169,7 @@ def test_check_style(self): 'GCC-4.9.2.eb', 'toy-0.0.eb', ] - stdout, _ = self._run_mock_eb(args, raise_error=True) + stdout, _ = self._run_mock_eb(args, raise_error=True, testing=True) regex = re.compile(r"Running style check on 2 easyconfig\(s\)", re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -3307,7 +3307,7 @@ def test_inject_checksums(self): self.assertEqual(stderr, '') args.append('--force') - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5' @@ -3396,7 +3396,7 @@ def test_inject_checksums(self): ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) @@ -3417,7 +3417,7 @@ def test_inject_checksums(self): write_file(test_ec, toy_ec_txt) args = [test_ec, '--inject-checksums=md5'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) patterns = [ "^== injecting md5 checksums in .*/test\.eb$", @@ -3479,7 +3479,7 @@ def test_force_download(self): '--force-download', '--sourcepath=%s' % self.test_prefix, ] - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, testing=True, strip=True) self.assertEqual(stdout, '') regex = re.compile("^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$") self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr)) From ef32ed299274b5f02e80408ab1ea8b2980e5890d Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 18:18:28 +0800 Subject: [PATCH 198/256] pass testing=True to _run_mock_eb in one more place --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 16a458f1f9..2015cbe3af 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3214,7 +3214,7 @@ def test_allow_use_as_root(self): # running as root is allowed under --allow-use-as-root, but does result in a warning being printed to stderr args = ['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences'] - _, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + _, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) expected = "WARNING: Using EasyBuild as root is NOT recommended, please proceed with care!\n" expected += "(this is only allowed because EasyBuild was configured with " From fa3b533ced51cb60a70d796c83f9b9063b73e01b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 20 Mar 2018 18:44:17 +0800 Subject: [PATCH 199/256] make default testing behaviour of _run_mock_eb consistent with that of eb_main --- test/framework/options.py | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 2015cbe3af..0208cd4baa 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2442,7 +2442,7 @@ def _assert_regexs(self, regexs, txt, assert_true=True): else: self.assertFalse(regex.search(txt), "Pattern '%s' NOT found in: %s" % (regex.pattern, txt)) - def _run_mock_eb(self, args, do_build=False, raise_error=False, verbose=False, testing=False, strip=False): + def _run_mock_eb(self, args, do_build=False, raise_error=False, verbose=False, testing=True, strip=False): """Helper function to mock easybuild runs""" self.mock_stdout(True) self.mock_stderr(True) @@ -2491,7 +2491,7 @@ def test_new_update_pr(self): '-D', '--disable-cleanup-tmpdir', ] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) # determine location of repo clone, can be used to test --git-working-dirs-path (and save time) dirs = glob.glob(os.path.join(self.test_prefix, 'eb-*', '*', 'git-working-dir*')) @@ -2528,7 +2528,7 @@ def test_new_update_pr(self): # add required commit message, try again args.append('--pr-commit-msg=just a test') - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs[-1] = r"^\s*2 files changed" regexs.remove(r"^\* title: \"\{tools\}\[gompi/1.3.12\] toy v0.0\"") @@ -2548,7 +2548,7 @@ def test_new_update_pr(self): '--pr-target-account=boegel', # we need to be able to 'clone' from here (via https) '--pr-title=test-1-2-3', ]) - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ r"^== fetching branch 'master' from https://github.com/boegel/easybuild-easyconfigs.git...", @@ -2584,7 +2584,7 @@ def test_new_update_pr(self): # also specifying commit message is sufficient; PR title is inherited from commit message args.append('--pr-commit-msg=this is just a test') - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regex = re.compile('^\* title: "this is just a test"', re.M) self.assertTrue(regex.search(txt), "Pattern '%s' is found in: %s" % (regex.pattern, txt)) @@ -2606,7 +2606,7 @@ def test_new_update_pr(self): self.mock_stdout(False) args.append('--pr-commit-msg="just a test"') - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ r"^== Determined branch name corresponding to easybuilders/easybuild-easyconfigs PR #2237: develop", @@ -2622,7 +2622,7 @@ def test_new_update_pr(self): args.remove('-D') args.append('-x') - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs.extend([ r"Full patch:", @@ -2655,7 +2655,7 @@ def test_new_pr_delete(self): '--pr-title=delete bzip2-1.6.0', '--pr-commit-msg="delete bzip2-1.6.0.eb"' ] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git...", @@ -2701,7 +2701,7 @@ def test_new_pr_dependencies(self): '-r%s' % self.test_prefix, ] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ r"^\* overview of changes:", @@ -2727,7 +2727,7 @@ def test_merge_pr(self): ] # merged PR for EasyBuild-3.3.0.eb, is missing approved review - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", @@ -2747,7 +2747,7 @@ def test_merge_pr(self): # full eligible merged PR args[1] = '4832' - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4832 for merging...", @@ -2774,7 +2774,7 @@ def test_merge_pr(self): '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, ] - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", @@ -2836,7 +2836,7 @@ def test_show_config(self): write_file(cfgfile, cfgtxt) args = ['--configfiles=%s' % cfgfile, '--show-config', '--buildpath=/weird/build/dir'] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) default_prefix = os.path.join(os.environ['HOME'], '.local', 'easybuild') @@ -2861,7 +2861,7 @@ def test_show_config(self): self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) args = ['--configfiles=%s' % cfgfile, '--show-full-config', '--buildpath=/weird/build/dir'] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) # output of --show-full-config includes additional lines for options with default values expected_lines.extend([ @@ -2878,7 +2878,7 @@ def test_show_config(self): # (existing config files are ignored via $EASYBUILD_IGNORECONFIGFILES) self.assertFalse(os.environ.get('EASYBUILD_CONFIGFILES', False)) args = ['--show-config', '--buildpath=/weird/build/dir'] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) self.assertTrue(re.search(r"buildpath\s* \(C\) = /weird/build/dir", txt)) # --show-config should not break including of easyblocks via $EASYBUILD_INCLUDE_EASYBLOCKS (see bug #1696) @@ -2893,7 +2893,7 @@ def test_show_config(self): os.environ['EASYBUILD_INCLUDE_EASYBLOCKS'] = testeasyblocktoinclude args = ['--show-config'] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) regex = re.compile(r'^include-easyblocks \(E\) = .*/testeasyblocktoinclude.py$', re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -2906,7 +2906,7 @@ def test_dump_env_config(self): args = ['%s.eb' % ec for ec in [fftw, gcc, openmpi]] + ['--dump-env-script'] os.chdir(self.test_prefix) - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) for name in [fftw, gcc, openmpi]: # check stdout @@ -2925,7 +2925,7 @@ def test_dump_env_config(self): os.chdir(self.test_prefix) args.append('--force') - self._run_mock_eb(args, do_build=True, raise_error=True, testing=True) + self._run_mock_eb(args, do_build=True, raise_error=True) # check contents of script env_script = os.path.join(self.test_prefix, '%s.env' % openmpi) @@ -2952,7 +2952,7 @@ def test_dump_env_config(self): def test_stop(self): """Test use of --stop.""" args = ['toy-0.0.eb', '--force', '--stop=configure'] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, strip=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) regex = re.compile("COMPLETED: Installation STOPPED successfully", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) @@ -3067,7 +3067,7 @@ def test_list_software(self): '--list-software', '--robot-paths=%s' % test_ecs, ] - txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True) + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected = '\n'.join([ "== Processed 5/5 easyconfigs...", "== Found 2 different software packages", @@ -3083,7 +3083,7 @@ def test_list_software(self): '--output-format=rst', '--robot-paths=%s' % test_ecs, ] - txt, _ = self._run_mock_eb(args) + txt, _ = self._run_mock_eb(args, testing=False) self.assertTrue(re.search('^\*GCC\*', txt, re.M)) self.assertTrue(re.search('^``4.6.3``\s+``dummy``', txt, re.M)) self.assertTrue(re.search('^\*gzip\*', txt, re.M)) @@ -3094,7 +3094,7 @@ def test_list_software(self): '--output-format=rst', '--robot-paths=%s' % test_ecs, ] - txt, _ = self._run_mock_eb(args) + txt, _ = self._run_mock_eb(args, testing=False) self.assertTrue(re.search('== Processed 5/5 easyconfigs...', txt, re.M)) self.assertTrue(re.search('== Found 2 different software packages', txt, re.M)) self.assertTrue(re.search('== Retained 1 installed software packages', txt, re.M)) @@ -3105,7 +3105,7 @@ def test_list_software(self): '--list-installed-software=detailed', '--robot-paths=%s' % test_ecs, ] - txt, _ = self._run_mock_eb(args) + txt, _ = self._run_mock_eb(args, testing=False) self.assertTrue(re.search('^== Retained 1 installed software packages', txt, re.M)) self.assertTrue(re.search('^\* GCC', txt, re.M)) self.assertTrue(re.search('^\s+\* GCC v4.6.3: dummy', txt, re.M)) @@ -3169,7 +3169,7 @@ def test_check_style(self): 'GCC-4.9.2.eb', 'toy-0.0.eb', ] - stdout, _ = self._run_mock_eb(args, raise_error=True, testing=True) + stdout, _ = self._run_mock_eb(args, raise_error=True) regex = re.compile(r"Running style check on 2 easyconfig\(s\)", re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -3214,7 +3214,7 @@ def test_allow_use_as_root(self): # running as root is allowed under --allow-use-as-root, but does result in a warning being printed to stderr args = ['toy-0.0.eb', '--allow-use-as-root-and-accept-consequences'] - _, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) + _, stderr = self._run_mock_eb(args, raise_error=True, strip=True) expected = "WARNING: Using EasyBuild as root is NOT recommended, please proceed with care!\n" expected += "(this is only allowed because EasyBuild was configured with " @@ -3307,7 +3307,7 @@ def test_inject_checksums(self): self.assertEqual(stderr, '') args.append('--force') - stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) toy_source_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' toy_patch_sha256 = '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5' @@ -3396,7 +3396,7 @@ def test_inject_checksums(self): ectxt = ectxt.replace(chksum, chksum[::-1]) write_file(test_ec, ectxt) - stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) ec = EasyConfigParser(test_ec).get_config_dict() self.assertEqual(ec['checksums'], [toy_source_sha256, toy_patch_sha256]) @@ -3417,7 +3417,7 @@ def test_inject_checksums(self): write_file(test_ec, toy_ec_txt) args = [test_ec, '--inject-checksums=md5'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, testing=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) patterns = [ "^== injecting md5 checksums in .*/test\.eb$", @@ -3479,7 +3479,7 @@ def test_force_download(self): '--force-download', '--sourcepath=%s' % self.test_prefix, ] - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, testing=True, strip=True) + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True) self.assertEqual(stdout, '') regex = re.compile("^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$") self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr)) From 36da24506a4f2b4c291545747011e0b05f1e7b2e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Mar 2018 10:34:53 +0100 Subject: [PATCH 200/256] specify GitHub login ID used in tests to --check-github in Travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2f30c5c19f..aa3cf0cd38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ script: # this is done to catch possible packaging issues - cd $HOME # check GitHub configuration - - eb --check-github + - eb --check-github --github-user=easybuild_test # run test suite - python -O -m test.framework.suite # check bootstrap script version From 1e40cf33a0874dd81688456ca25009ad01405d05 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 22 Mar 2018 11:42:43 +0100 Subject: [PATCH 201/256] Add support for a new toolchain, gomkl. GCC + OpenMPI + MKL. --- easybuild/toolchains/gomkl.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 easybuild/toolchains/gomkl.py diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py new file mode 100644 index 0000000000..04b2dbbab1 --- /dev/null +++ b/easybuild/toolchains/gomkl.py @@ -0,0 +1,42 @@ +## +# Copyright 2012-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gomkl compiler toolchain (includes GCC, OpenMPI, +Intel Math Kernel Library (MKL) and Intel FFTW wrappers). + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Ake Sandgren (Umea University) +""" + +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Gomkl(Gompi, IntelMKL, IntelFFTW): + """Compiler toolchain with GCC, OpenMPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers.""" + NAME = 'gomkl' + SUBTOOLCHAIN = Gompi.NAME From 6b32291147798115f2ea61851a35cce677bc9e1a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 23 Mar 2018 11:11:38 +0100 Subject: [PATCH 202/256] use DEFAULT_OPT_LEVEL constant + fix minor style issues --- easybuild/toolchains/compiler/gcc.py | 8 ++++---- test/framework/toolchain.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 02e5aec394..ddce7b6962 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -35,7 +35,7 @@ import easybuild.tools.systemtools as systemtools from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.toolchain.compiler import Compiler +from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL TC_CONSTANT_GCC = "GCC" @@ -61,11 +61,11 @@ class Gcc(Compiler): 'lto': 'flto', 'ieee': ['mieee-fp', 'fno-trapping-math'], 'strict': ['mieee-fp', 'mno-recip'], - 'precise':['mno-recip'], - 'defaultprec':['fno-math-errno'], + 'precise': ['mno-recip'], + 'defaultprec': ['fno-math-errno'], 'loose': ['fno-math-errno', 'mrecip', 'mno-ieee-fp'], 'veryloose': ['fno-math-errno', 'mrecip=all', 'mno-ieee-fp'], - 'defaultopt': ['O2', 'ftree-vectorize'], + DEFAULT_OPT_LEVEL: ['O2', 'ftree-vectorize'], } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0bacbf7a87..effe171b0b 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -579,8 +579,10 @@ def test_goolfc(self): tc.set_options(opts) tc.prepare() + archflags = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)] + optflags = "-O2 -ftree-vectorize -%s -fno-math-errno -fopenmp" % archflags nvcc_flags = r' '.join([ - r'-Xcompiler="-O2 -ftree-vectorize -%s -fno-math-errno -fopenmp"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)], + r'-Xcompiler="%s"' % optflags, # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), From 8e02128635e5953feb29a2b500b9237acaf4625b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sun, 25 Mar 2018 01:31:36 +0000 Subject: [PATCH 203/256] Rename has_depends_on to supports_depends_on and init properly. Now initialized to False by default and only set depending on version in Lmod.__init__, as suggested by @boegel. --- easybuild/tools/module_generator.py | 4 ++-- easybuild/tools/modules.py | 8 +++----- test/framework/module_generator.py | 4 ++-- test/framework/options.py | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index dc87e23b37..ae55d6281d 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -509,7 +509,7 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): load_template = self.LOAD_TEMPLATE # Lmod 7.6.1+ supports depends-on which does this most nicely: if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on') and modules_tool().has_depends_on: + recursive_unload == 'depends_on') and modules_tool().supports_depends_on: load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) @@ -807,7 +807,7 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on') and modules_tool().has_depends_on: + recursive_unload == 'depends_on') and modules_tool().supports_depends_on: load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 6e7709b319..6ee00cfbf3 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -197,6 +197,7 @@ def __init__(self, mod_paths=None, testing=False): self.check_module_path() self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch')) self.set_and_check_version() + self.supports_depends_on = False def buildstats(self): """Return tuple with data to be included in buildstats""" @@ -209,11 +210,8 @@ def modules(self): def set_and_check_version(self): """Get the module version, and check any requirements""" - lmod_depends_on_min = '7.6.1' if self.COMMAND in MODULE_VERSION_CACHE: self.version = MODULE_VERSION_CACHE[self.COMMAND] - self.has_depends_on = (isinstance(self, Lmod) and - StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min)) self.log.debug("Found cached version for %s: %s", self.COMMAND, self.version) return @@ -262,8 +260,6 @@ def set_and_check_version(self): self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION) MODULE_VERSION_CACHE[self.COMMAND] = self.version - self.has_depends_on = (isinstance(self, Lmod) and - StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min)) def check_cmd_avail(self): """Check whether modules tool command is available.""" @@ -1106,6 +1102,7 @@ class Lmod(ModulesTool): COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' REQ_VERSION = '5.8' + REQ_VERSION_DEPENDS_ON = '7.6.1' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') @@ -1121,6 +1118,7 @@ def __init__(self, *args, **kwargs): setvar('LMOD_REDIRECT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) + self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON) def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 5509729040..e53e509db9 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -260,7 +260,7 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name")) # Lmod 7.6+ depends-on - if self.modtool.has_depends_on: + if self.modtool.supports_depends_on: expected = '\n'.join([ '', "depends-on mod_name", @@ -295,7 +295,7 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name")) # Lmod 7.6+ depends_on - if self.modtool.has_depends_on: + if self.modtool.supports_depends_on: expected = '\n'.join([ '', 'depends_on("mod_name")', diff --git a/test/framework/options.py b/test/framework/options.py index 26470b0d84..d7b6762b95 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1157,7 +1157,7 @@ def test_recursive_module_unload(self): # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] - if self.modtool.has_depends_on: + if self.modtool.supports_depends_on: lastargs.append(lastargs[0]+'-depends-on') for lastarg in lastargs: args = [ From 5e2970270ff66d4d8ff8e1d56897d4ef2b074edc Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Mar 2018 01:04:17 +0000 Subject: [PATCH 204/256] Rename recursive-module-unload-depends-on to module-use-depends-on. Decouple module-use-depends-on completely from recursive-module-unload, and change recursive-module-unload-depends-on to module-use-depends-on as suggested by @boegel. --- easybuild/framework/easyblock.py | 10 +++++----- easybuild/framework/easyconfig/default.py | 3 ++- easybuild/tools/config.py | 4 ++-- easybuild/tools/module_generator.py | 14 ++++++++------ easybuild/tools/options.py | 5 +++-- test/framework/module_generator.py | 8 ++++---- test/framework/options.py | 2 +- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 48269647ea..0085e290bb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -937,8 +937,7 @@ def make_devel_module(self, create_in_builddir=False): # capture all the EBDEVEL vars # these should be all the dependencies and we should load them recursive_unload = self.cfg['recursive_module_unload'] - if self.cfg['recursive_module_unload_depends_on']: - recursive_unload = 'depends_on' + use_depends_on = self.cfg['module_use_depends_on'] for key in os.environ: # legacy support if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): @@ -946,7 +945,8 @@ def make_devel_module(self, create_in_builddir=False): path = os.environ[key] if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] - load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload) + load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload, + use_depends_on=use_depends_on) load_lines.append(load_statement) elif key.startswith('SOFTDEVEL'): self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') @@ -1045,14 +1045,14 @@ def make_module_dep(self, unload_info=None): # include load statements for retained dependencies recursive_unload = self.cfg['recursive_module_unload'] - if self.cfg['recursive_module_unload_depends_on']: - recursive_unload = 'depends_on' + use_depends_on = self.cfg['module_use_depends_on'] loads = [] for dep in deps: unload_modules = [] if dep in unload_info: unload_modules.append(unload_info[dep]) loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload, + use_depends_on=use_depends_on, unload_modules=unload_modules)) # Force unloading any other modules diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 9e0c64ecb1..1054bec961 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -171,8 +171,9 @@ 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], + 'module_use_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ', + "(implies recursive unloading of modules).", MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], - 'recursive_module_unload_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module', MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index cd9b99380c..0d76e1f4bf 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -236,7 +236,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): False: [ 'dry_run', 'recursive_mod_unload', - 'recursive_mod_unload_depends_on', + 'mod_use_depends_on', 'retain_all_deps', 'silent', 'try_to_generate', @@ -387,7 +387,7 @@ def init_build_options(build_options=None, cmdline_options=None): 'check_osdeps': not cmdline_options.ignore_osdeps, 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, 'recursive_mod_unload': cmdline_options.recursive_module_unload, - 'recursive_mod_unload_depends_on': cmdline_options.recursive_module_unload_depends_on, + 'mod_use_depends_on': cmdline_options.module_use_depends_on, 'retain_all_deps': retain_all_deps, 'validate': not cmdline_options.force, 'valid_module_classes': module_classes(), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ae55d6281d..925eb6a6fe 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -495,7 +495,7 @@ def getenv_cmd(self, envvar): """ return '$env(%s)' % envvar - def load_module(self, mod_name, recursive_unload=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -508,8 +508,9 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body.extend([self.unload_module(m).strip() for m in unload_modules]) load_template = self.LOAD_TEMPLATE # Lmod 7.6.1+ supports depends-on which does this most nicely: - if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on') and modules_tool().supports_depends_on: + if build_option('mod_use_depends_on') or use_depends_on: + if not modules_tool().supports_depends_on: + raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) @@ -792,7 +793,7 @@ def getenv_cmd(self, envvar): """ return 'os.getenv("%s")' % envvar - def load_module(self, mod_name, recursive_unload=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -806,8 +807,9 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: - if (build_option('recursive_mod_unload_depends_on') or - recursive_unload == 'depends_on') and modules_tool().supports_depends_on: + if build_option('mod_use_depends_on') or use_depends_on: + if not modules_tool().supports_depends_on: + raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8a419b0ee3..ea62c5296e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -456,6 +456,9 @@ def config_options(self): 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), + 'module-use-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules " + "(implies recursive unloading of modules).", + None, 'store_true', False), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), 'strlist', 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), @@ -474,8 +477,6 @@ def config_options(self): None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), - 'recursive-module-unload-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules.", - None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index e53e509db9..d8c62aa722 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -266,8 +266,8 @@ def test_load(self): "depends-on mod_name", '', ]) - self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on")) - init_config(build_options={'recursive_mod_unload_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name", use_depends_on=True)) + init_config(build_options={'mod_use_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) else: # default: guarded module load (which implies no recursive unloading) @@ -301,8 +301,8 @@ def test_load(self): 'depends_on("mod_name")', '', ]) - self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on")) - init_config(build_options={'recursive_mod_unload_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name", use_depends_on=True)) + init_config(build_options={'mod_use_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) def test_unload(self): diff --git a/test/framework/options.py b/test/framework/options.py index d7b6762b95..0420db0812 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1158,7 +1158,7 @@ def test_recursive_module_unload(self): # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] if self.modtool.supports_depends_on: - lastargs.append(lastargs[0]+'-depends-on') + lastargs.append('--module-use-depends-on') for lastarg in lastargs: args = [ eb_file, From 279e4a32c465d01c6a74c1bdc6a81dae32632d5b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Mar 2018 02:40:26 +0000 Subject: [PATCH 205/256] Fix string typo in module_use_depends_on help. --- easybuild/framework/easyconfig/default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 1054bec961..1e3e078ce2 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -171,8 +171,8 @@ 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], - 'module_use_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ', - "(implies recursive unloading of modules).", MODULES], + 'module_use_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' + '(implies recursive unloading of modules).', MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], # MODULES documentation easyconfig parameters From 7e5fdaa419fe51a10aa9911d914e20e545297e4e Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Mon, 26 Mar 2018 18:15:24 +0200 Subject: [PATCH 206/256] NEW move mock to prod api --- easybuild/tools/modules.py | 10 ++++++++++ test/framework/modulestool.py | 11 +---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 62ae8db77c..2a642b8cad 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1384,3 +1384,13 @@ class Modules(EnvironmentModulesC): """NO LONGER SUPPORTED: interface to modules tool, use modules_tool from easybuild.tools.modules instead""" def __init__(self, *args, **kwargs): _log.nosupport("modules.Modules class is now an abstract interface, use modules.modules_tool instead", '2.0') + + +class MockModulesTool(modules.ModulesTool): + """ MockModule class""" + COMMAND = 'echo' + VERSION_OPTION = '1.0' + VERSION_REGEXP = r'(?P\d\S*)' + # redirect to stderr, ignore 'echo python' ($0 and $1) + COMMAND_SHELL = ["bash", "-c", "echo $2 $3 $4 1>&2"] + \ No newline at end of file diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 4c5adf0073..d5596eedaf 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -43,19 +43,10 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import which, write_file -from easybuild.tools.modules import modules_tool, Lmod +from easybuild.tools.modules import modules_tool, Lmod, MockModulesTool from test.framework.utilities import init_config -class MockModulesTool(modules.ModulesTool): - """ MockModule class""" - COMMAND = 'echo' - VERSION_OPTION = '1.0' - VERSION_REGEXP = r'(?P\d\S*)' - # redirect to stderr, ignore 'echo python' ($0 and $1) - COMMAND_SHELL = ["bash", "-c", "echo $2 $3 $4 1>&2"] - - class BrokenMockModulesTool(MockModulesTool): """MockModulesTool class that is broken unless environment command is set""" COMMAND = '/does/not/exist' From 09636635ffb73a9c142863798547a4b06ef355a9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Mar 2018 17:23:18 +0000 Subject: [PATCH 207/256] Introduce 'vectorize' toolchainopt. If set to True or False, will explicitly enable or disable auto-vectorization. Otherwise (if left to None) noopt and lowopt will not vectorize and other optimzations levels will vectorize. --- easybuild/toolchains/compiler/clang.py | 2 ++ easybuild/toolchains/compiler/gcc.py | 2 ++ easybuild/toolchains/compiler/ibmxl.py | 5 ++++- .../toolchains/compiler/inteliccifort.py | 2 ++ easybuild/toolchains/compiler/pgi.py | 2 ++ easybuild/tools/toolchain/compiler.py | 22 +++++++++++++++++-- test/framework/toolchain.py | 2 +- 7 files changed, 33 insertions(+), 4 deletions(-) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 8e1be062c9..36a23e4d17 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -85,6 +85,8 @@ class Clang(Compiler): 'defaultprec': [], 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], 'veryloose': ['ffast-math'], + 'vectorize': 'fvectorize', + 'novectorize': 'fno-vectorize', } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index ddce7b6962..6743576a38 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -65,6 +65,8 @@ class Gcc(Compiler): 'defaultprec': ['fno-math-errno'], 'loose': ['fno-math-errno', 'mrecip', 'mno-ieee-fp'], 'veryloose': ['fno-math-errno', 'mrecip=all', 'mno-ieee-fp'], + 'vectorize': 'ftree-vectorize', + 'novectorize': 'fno-tree-vectorize', DEFAULT_OPT_LEVEL: ['O2', 'ftree-vectorize'], } diff --git a/easybuild/toolchains/compiler/ibmxl.py b/easybuild/toolchains/compiler/ibmxl.py index b4efb3ce92..441a0eef4c 100644 --- a/easybuild/toolchains/compiler/ibmxl.py +++ b/easybuild/toolchains/compiler/ibmxl.py @@ -11,7 +11,7 @@ from distutils.version import LooseVersion import easybuild.tools.systemtools as systemtools -from easybuild.tools.toolchain.compiler import Compiler +from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL TC_CONSTANT_IBMCOMP = "IBMXL" @@ -35,6 +35,9 @@ class IBMXL(Compiler): 'defaultprec': ['', '', ''], 'loose': [''], 'veryloose': [''], + 'vectorize': 'qsimd=auto', + 'novectorize': 'qsimd=noauto', + DEFAULT_OPT_LEVEL: ['O2', 'qsimd=auto'], 'ibm-static': 'qstaticlink=xllibs', 'pic': 'qpic', 'shared': 'qmkshrobj', diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 0ec3cde9f1..052b223392 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -64,6 +64,8 @@ class IntelIccIfort(Compiler): 'defaultprec': ['ftz', 'fp-speculation=safe', 'fp-model source'], 'loose': ['fp-model fast=1'], 'veryloose': ['fp-model fast=2'], + 'vectorize': 'vec', + 'novectorize': 'no-vec', 'intel-static': 'static-intel', 'no-icc': 'no-icc', 'error-unknown-option': 'we10006', # error at warning #10006: ignoring unknown option diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py index f5ec0ec94f..f197ba069a 100644 --- a/easybuild/toolchains/compiler/pgi.py +++ b/easybuild/toolchains/compiler/pgi.py @@ -65,6 +65,8 @@ class Pgi(Compiler): 'defaultprec': ['Mflushz'], 'loose': ['Mfprelaxed'], 'veryloose': ['Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', 'Mfpapprox'], + 'vectorize': 'Mvect', + 'novectorize': 'Mnovect', } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 3147e9ba86..cc804a74b2 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -88,6 +88,8 @@ class Compiler(Toolchain): 'static': (False, "Build static library"), '32bit': (False, "Compile 32bit target"), # LA, FFTW 'openmp': (False, "Enable OpenMP"), + 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), + 'novectorize': (None, "Disable compiler auto-vectorization, default for noopt and lowopt"), # not set, only used to map 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), } @@ -246,7 +248,23 @@ def _set_compiler_flags(self): # 1st one is the one to use. add default at the end so len is at least 1 optflags = [self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] + \ - [self.options.option(default_opt_level)] + [self.options.option(default_opt_level)][:1] + + # only apply if the vectorize toolchainopt is explicitly set + # otherwise the individual compiler toolchain file should make sure that + # vectorization is disabled for noopt and lowopt, and enabled otherwise. + if self.options.get('vectorize') is not None: + novectorize = self.options.option('novectorize') + vectorize = self.options.option('vectorize') + if self.options['vectorize']: + vectflags = vectorize + else: + vectflags = novectorize + # avoid double use of such flags + for setting in novectorize, vectorize: + if setting in optflags[0]: + optflags[0].remove(setting) + optflags.append(vectflags) optarchflags = [] if build_option('optarch') == OPTARCH_GENERIC: @@ -259,7 +277,7 @@ def _set_compiler_flags(self): precflags = [self.options.option(x) for x in self.COMPILER_PREC_FLAGS if self.options.get(x, False)] + \ [self.options.option('defaultprec')] - self.variables.nextend('OPTFLAGS', optflags[:1] + optarchflags) + self.variables.nextend('OPTFLAGS', optflags + optarchflags) self.variables.nextend('PRECFLAGS', precflags[:1]) # precflags last diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index effe171b0b..dd480ed328 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -290,7 +290,7 @@ def test_misc_flags_unique(self): flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # setting option should result in corresponding flag to be set (unique options) - for opt in ['unroll', 'optarch', 'openmp']: + for opt in ['unroll', 'optarch', 'openmp', 'vectorize']: for enable in [True, False]: tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({opt: enable}) From 71dbf6d665f53ca2446662c219eb98994826771e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Mar 2018 18:19:36 +0000 Subject: [PATCH 208/256] We need a copy of optflags[0] instead of modifying in place. Modifying in place changes the original optflags which messes up further tests in the toolchain tests. --- easybuild/tools/toolchain/compiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index cc804a74b2..6252f28df9 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -261,9 +261,9 @@ def _set_compiler_flags(self): else: vectflags = novectorize # avoid double use of such flags - for setting in novectorize, vectorize: - if setting in optflags[0]: - optflags[0].remove(setting) + if isinstance(optflags[0], list): + optflags[0] = [setting for setting in optflags[0] + if setting not in (novectorize, vectorize)] optflags.append(vectflags) optarchflags = [] From 6a962a24a4053d4f1738e5cfb84027d565fe0e21 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 26 Mar 2018 22:55:05 +0000 Subject: [PATCH 209/256] Use dictionary with boolean keys for 'vectorize' toolchainopt. This avoids the need for 'novectorize' and makes for more compact logic. --- easybuild/toolchains/compiler/clang.py | 3 +-- easybuild/toolchains/compiler/gcc.py | 3 +-- easybuild/toolchains/compiler/ibmxl.py | 3 +-- easybuild/toolchains/compiler/inteliccifort.py | 3 +-- easybuild/toolchains/compiler/pgi.py | 3 +-- easybuild/tools/toolchain/compiler.py | 11 ++--------- test/framework/toolchain.py | 2 ++ 7 files changed, 9 insertions(+), 19 deletions(-) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 36a23e4d17..b877fcb31c 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -85,8 +85,7 @@ class Clang(Compiler): 'defaultprec': [], 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], 'veryloose': ['ffast-math'], - 'vectorize': 'fvectorize', - 'novectorize': 'fno-vectorize', + 'vectorize': {False: 'fno-vectorize', True: 'fvectorize'}, } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 6743576a38..d92aec0191 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -65,8 +65,7 @@ class Gcc(Compiler): 'defaultprec': ['fno-math-errno'], 'loose': ['fno-math-errno', 'mrecip', 'mno-ieee-fp'], 'veryloose': ['fno-math-errno', 'mrecip=all', 'mno-ieee-fp'], - 'vectorize': 'ftree-vectorize', - 'novectorize': 'fno-tree-vectorize', + 'vectorize': {False: 'fno-tree-vectorize', True: 'ftree-vectorize'}, DEFAULT_OPT_LEVEL: ['O2', 'ftree-vectorize'], } diff --git a/easybuild/toolchains/compiler/ibmxl.py b/easybuild/toolchains/compiler/ibmxl.py index 441a0eef4c..65170d6ad7 100644 --- a/easybuild/toolchains/compiler/ibmxl.py +++ b/easybuild/toolchains/compiler/ibmxl.py @@ -35,8 +35,7 @@ class IBMXL(Compiler): 'defaultprec': ['', '', ''], 'loose': [''], 'veryloose': [''], - 'vectorize': 'qsimd=auto', - 'novectorize': 'qsimd=noauto', + 'vectorize': {False: 'qsimd=noauto', True: 'qsimd=auto'}, DEFAULT_OPT_LEVEL: ['O2', 'qsimd=auto'], 'ibm-static': 'qstaticlink=xllibs', 'pic': 'qpic', diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 052b223392..86b2330ae8 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -64,8 +64,7 @@ class IntelIccIfort(Compiler): 'defaultprec': ['ftz', 'fp-speculation=safe', 'fp-model source'], 'loose': ['fp-model fast=1'], 'veryloose': ['fp-model fast=2'], - 'vectorize': 'vec', - 'novectorize': 'no-vec', + 'vectorize': {False: 'no-vec', True: 'vec'}, 'intel-static': 'static-intel', 'no-icc': 'no-icc', 'error-unknown-option': 'we10006', # error at warning #10006: ignoring unknown option diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py index f197ba069a..338789be69 100644 --- a/easybuild/toolchains/compiler/pgi.py +++ b/easybuild/toolchains/compiler/pgi.py @@ -65,8 +65,7 @@ class Pgi(Compiler): 'defaultprec': ['Mflushz'], 'loose': ['Mfprelaxed'], 'veryloose': ['Mfprelaxed=div,order,intrinsic,recip,sqrt,rsqrt', 'Mfpapprox'], - 'vectorize': 'Mvect', - 'novectorize': 'Mnovect', + 'vectorize': {False: 'Mnovect', True: 'Mvect'}, } # used when 'optarch' toolchain option is enabled (and --optarch is not specified) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 6252f28df9..66d1f83e6f 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -89,7 +89,6 @@ class Compiler(Toolchain): '32bit': (False, "Compile 32bit target"), # LA, FFTW 'openmp': (False, "Enable OpenMP"), 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), - 'novectorize': (None, "Disable compiler auto-vectorization, default for noopt and lowopt"), # not set, only used to map 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), } @@ -254,16 +253,10 @@ def _set_compiler_flags(self): # otherwise the individual compiler toolchain file should make sure that # vectorization is disabled for noopt and lowopt, and enabled otherwise. if self.options.get('vectorize') is not None: - novectorize = self.options.option('novectorize') - vectorize = self.options.option('vectorize') - if self.options['vectorize']: - vectflags = vectorize - else: - vectflags = novectorize + vectflags = self.options.option('vectorize')[self.options['vectorize']] # avoid double use of such flags if isinstance(optflags[0], list): - optflags[0] = [setting for setting in optflags[0] - if setting not in (novectorize, vectorize)] + optflags[0] = [flag for flag in optflags[0] if flag not in self.options.option('vectorize').values()] optflags.append(vectflags) optarchflags = [] diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index dd480ed328..81677a0c10 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -297,6 +297,8 @@ def test_misc_flags_unique(self): tc.prepare() if opt == 'optarch': flag = '-%s' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)] + elif isinstance(tc.options.options_map[opt], dict): + flag = '-%s' % tc.options.options_map[opt][True] else: flag = '-%s' % tc.options.options_map[opt] for var in flag_vars: From 9acce35928c22a02e1ab45f7ace77630260a5307 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Tue, 27 Mar 2018 11:39:06 +0200 Subject: [PATCH 210/256] NEW woriking --fetch option --- easybuild/main.py | 6 +++++- easybuild/tools/modules.py | 15 +++++++++++++-- easybuild/tools/options.py | 10 +++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 02345fb867..98ac3fd50c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -62,7 +62,7 @@ from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook -from easybuild.tools.modules import modules_tool +from easybuild.tools.modules import modules_tool, MockModulesTool from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support @@ -255,6 +255,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) + if options.fetch: + _log.debug("Fetching sources only, implies stop=fetch, moduletool=MockModulesTool, ignore-osdep.") + modtool = MockModulesTool(testing=testing) + if modtool is None: modtool = modules_tool(testing=testing) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 2a642b8cad..6fd965848f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1386,11 +1386,22 @@ def __init__(self, *args, **kwargs): _log.nosupport("modules.Modules class is now an abstract interface, use modules.modules_tool instead", '2.0') -class MockModulesTool(modules.ModulesTool): +class MockModulesTool(ModulesTool): """ MockModule class""" COMMAND = 'echo' VERSION_OPTION = '1.0' VERSION_REGEXP = r'(?P\d\S*)' # redirect to stderr, ignore 'echo python' ($0 and $1) COMMAND_SHELL = ["bash", "-c", "echo $2 $3 $4 1>&2"] - \ No newline at end of file + + def check_loaded_modules(self): + """Nothing to do since no modules""" + pass + + def list(self): + """No modules loaded""" + return [] + + def update(self): + """Update after new modules were added.""" + pass diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 02b8874c14..30b770a314 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -50,7 +50,7 @@ from vsc.utils.missing import nub import easybuild.tools.environment as env -from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, EasyBlock +from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, FETCH_STEP, EasyBlock from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8 from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict @@ -255,6 +255,8 @@ def basic_options(self): 'extended-dry-run': ("Print build environment and (expected) build procedure that will be performed", None, 'store_true', False, 'x'), 'extended-dry-run-ignore-errors': ("Ignore errors that occur during dry run", None, 'store_true', True), + 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " + "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module), " "and skipping check for OS dependencies", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), @@ -857,6 +859,12 @@ def _postprocess_config(self): if self.options.search_paths is not None: self.options.search_paths = [os.path.abspath(path) for path in self.options.search_paths] + # Implies stop=fetch, moduletool=MockModulesTool, ignore-osdeps and no lua module syntax + if self.options.fetch: + self.options.stop = FETCH_STEP + self.options.ignore_osdeps = True + self.options.modules_tool = None + def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' From 153e9bcd1bf9633dfeee1e492fd55f0d1e8eec8b Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Tue, 27 Mar 2018 11:44:32 +0200 Subject: [PATCH 211/256] IMPROVE do not propose MockModulesTool as modulestool --- easybuild/tools/modules.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 6fd965848f..430eca21b8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1340,6 +1340,8 @@ def avail_modules_tools(): # filter out legacy Modules class if 'Modules' in class_dict: del class_dict['Modules'] + if 'MockModulesTool' in class_dict: + del class_dict['MockModulesTool'] return class_dict From 9360ead74acbfbfe87b301c226b329a72d74287f Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Tue, 27 Mar 2018 11:45:59 +0200 Subject: [PATCH 212/256] CLEAN add code documentation --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 430eca21b8..3b7f0dee6a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1389,7 +1389,7 @@ def __init__(self, *args, **kwargs): class MockModulesTool(ModulesTool): - """ MockModule class""" + """Class that mock the module behaviour, used for operation not requiring modules. Eg. tests, fetch only""" COMMAND = 'echo' VERSION_OPTION = '1.0' VERSION_REGEXP = r'(?P\d\S*)' From 6d27024262deaf658918ec9d80dc1efdd3e25099 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Tue, 27 Mar 2018 12:37:38 +0200 Subject: [PATCH 213/256] FIX revert using MoclModulesTool from test, use a dedicated one it broke tests --- easybuild/main.py | 6 +++--- easybuild/tools/modules.py | 6 +++--- easybuild/tools/options.py | 2 +- test/framework/modulestool.py | 11 ++++++++++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 98ac3fd50c..10b6f2572c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -62,7 +62,7 @@ from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook -from easybuild.tools.modules import modules_tool, MockModulesTool +from easybuild.tools.modules import modules_tool, NoModulesTool from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support @@ -256,8 +256,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): config.init_build_options(build_options=build_options, cmdline_options=options) if options.fetch: - _log.debug("Fetching sources only, implies stop=fetch, moduletool=MockModulesTool, ignore-osdep.") - modtool = MockModulesTool(testing=testing) + _log.debug("Fetching sources only, implies stop=fetch, moduletool=NoModulesTool, ignore-osdep.") + modtool = NoModulesTool(testing=testing) if modtool is None: modtool = modules_tool(testing=testing) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 3b7f0dee6a..45c3e69df6 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1340,8 +1340,8 @@ def avail_modules_tools(): # filter out legacy Modules class if 'Modules' in class_dict: del class_dict['Modules'] - if 'MockModulesTool' in class_dict: - del class_dict['MockModulesTool'] + if 'NoModulesTool' in class_dict: + del class_dict['NoModulesTool'] return class_dict @@ -1388,7 +1388,7 @@ def __init__(self, *args, **kwargs): _log.nosupport("modules.Modules class is now an abstract interface, use modules.modules_tool instead", '2.0') -class MockModulesTool(ModulesTool): +class NoModulesTool(ModulesTool): """Class that mock the module behaviour, used for operation not requiring modules. Eg. tests, fetch only""" COMMAND = 'echo' VERSION_OPTION = '1.0' diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 30b770a314..dbcf5b12fc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -859,7 +859,7 @@ def _postprocess_config(self): if self.options.search_paths is not None: self.options.search_paths = [os.path.abspath(path) for path in self.options.search_paths] - # Implies stop=fetch, moduletool=MockModulesTool, ignore-osdeps and no lua module syntax + # Implies stop=fetch, moduletool=NoModulesTool, ignore-osdeps and no lua module syntax if self.options.fetch: self.options.stop = FETCH_STEP self.options.ignore_osdeps = True diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index d5596eedaf..4c5adf0073 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -43,10 +43,19 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import which, write_file -from easybuild.tools.modules import modules_tool, Lmod, MockModulesTool +from easybuild.tools.modules import modules_tool, Lmod from test.framework.utilities import init_config +class MockModulesTool(modules.ModulesTool): + """ MockModule class""" + COMMAND = 'echo' + VERSION_OPTION = '1.0' + VERSION_REGEXP = r'(?P\d\S*)' + # redirect to stderr, ignore 'echo python' ($0 and $1) + COMMAND_SHELL = ["bash", "-c", "echo $2 $3 $4 1>&2"] + + class BrokenMockModulesTool(MockModulesTool): """MockModulesTool class that is broken unless environment command is set""" COMMAND = '/does/not/exist' From c2aca6692c97b0b06e9392445bea325b284ed309 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 27 Mar 2018 13:39:04 +0000 Subject: [PATCH 214/256] Introduce golf: compiler toolchain with GCC, OpenBLAS, and FFTW. Since BLACS_MODULE_NAME and SCALAPACK_MODULE_NAME are set to None in this case we need to filter out None for the toolchain definition. --- easybuild/toolchains/golf.py | 40 ++++++++++++++++++++++++++ easybuild/tools/toolchain/toolchain.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 easybuild/toolchains/golf.py diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py new file mode 100644 index 0000000000..32ce7ecd42 --- /dev/null +++ b/easybuild/toolchains/golf.py @@ -0,0 +1,40 @@ +## +# Copyright 2013-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for golf compiler toolchain (includes GCC, OpenBLAS, LAPACK, and FFTW). + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS + + +class Golf(GccToolchain, OpenBLAS, Fftw): + """Compiler toolchain with GCC, OpenBLAS, and FFTW.""" + NAME = 'golf' + SUBTOOLCHAIN = GccToolchain.NAME diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 18e5f8ed82..0ca244d131 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -412,7 +412,7 @@ def definition(self): var_suff = '_MODULE_NAME' tc_elems = {} for var in dir(self): - if var.endswith(var_suff): + if var.endswith(var_suff) and getattr(self, var) is not None: tc_elems.update({var[:-len(var_suff)]: getattr(self, var)}) self.log.debug("Toolchain definition for %s: %s", self.as_dict(), tc_elems) From f64ba7720e20e32ca51b59c43999ab6f258af6b9 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Wed, 28 Mar 2018 17:23:54 +0200 Subject: [PATCH 215/256] CLEAN comments --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index dbcf5b12fc..8198be9c72 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -859,7 +859,7 @@ def _postprocess_config(self): if self.options.search_paths is not None: self.options.search_paths = [os.path.abspath(path) for path in self.options.search_paths] - # Implies stop=fetch, moduletool=NoModulesTool, ignore-osdeps and no lua module syntax + # Fetch option implies stop=fetch, no moduletool and ignore-osdeps if self.options.fetch: self.options.stop = FETCH_STEP self.options.ignore_osdeps = True From e4ebbb2cd739a476bd9bed98f78a11e3f356eb8c Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 10:06:32 +0200 Subject: [PATCH 216/256] CLEAN NoModulesTool implementation and add tests --- easybuild/tools/modules.py | 19 ++++++++++--------- test/framework/modules.py | 9 ++++++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 45c3e69df6..9dce50b668 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1390,12 +1390,13 @@ def __init__(self, *args, **kwargs): class NoModulesTool(ModulesTool): """Class that mock the module behaviour, used for operation not requiring modules. Eg. tests, fetch only""" - COMMAND = 'echo' - VERSION_OPTION = '1.0' - VERSION_REGEXP = r'(?P\d\S*)' - # redirect to stderr, ignore 'echo python' ($0 and $1) - COMMAND_SHELL = ["bash", "-c", "echo $2 $3 $4 1>&2"] - + def __init__(self, *args, **kwargs): + pass + + def exist(self, mod_names, *args, **kwargs): + """No modules, so nothing exists""" + return [False]*len(mod_names) + def check_loaded_modules(self): """Nothing to do since no modules""" pass @@ -1404,6 +1405,6 @@ def list(self): """No modules loaded""" return [] - def update(self): - """Update after new modules were added.""" - pass + def available(self, *args, **kwargs): + """No modules, so nothing available""" + return [] diff --git a/test/framework/modules.py b/test/framework/modules.py index 8b57398272..f4f2daf421 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -45,7 +45,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, write_file -from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesTcl, Lmod +from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches from easybuild.tools.run import run_cmd @@ -929,6 +929,13 @@ def check_loaded_modules(): error_msg = "Unknown action specified to --detect-loaded-modules: sdvbfdgh" self.assertErrorRegex(EasyBuildError, error_msg, init_config, args=['--detect-loaded-modules=sdvbfdgh']) + def test_NoModulesTool(self): + nmt = NoModulesTool(testing=true) + assertEqual(len(nmt.available()), 0) + assertEqual(len(nmt.available(mod_names='foo')), 0) + assertEqual(len(nmt.list()), 0) + assertEqual(nmt.exist(['foo', 'bar']), [False, False]) + assertEqual(nmt.exist(['foo', 'bar'], r'^\s*\S*/%s.*:\s*$', skip_avail=False), [False, False]) def suite(): """ returns all the testcases in this module """ From e2cf7057a4c63856783b40129a0d2e31abaa7918 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 10:25:52 +0200 Subject: [PATCH 217/256] NEW option fetch tests --- test/framework/options.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 780635eb33..0037e2a7c4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -40,7 +40,7 @@ import easybuild.tools.build_log import easybuild.tools.options import easybuild.tools.toolchain -from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyblock import EasyBlock, FETCH_STEP from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -2957,6 +2957,28 @@ def test_stop(self): regex = re.compile("COMPLETED: Installation STOPPED successfully", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_fetch(self): + options = EasyBuildOptions(go_args=['--fetch']) + options.postprocess() + + self.assertEqual(options.stop, FETCH_STEP) + self.assertEqual(options.modulestool, None) + self.assertTrue(options.ignore_osdeps) + + args = ['toy-0.0.eb', '--fetch'] + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + + patterns = [ + "^== fetching files\.\.\.$" + "^== COMPLETED: Installation STOPPED successfully$" + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) + + regex = re.compile("^== creating build dir, resetting environment\.\.\.$") + self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def test_parse_external_modules_metadata(self): """Test parse_external_modules_metadata function.""" # by default, provided external module metadata cfg files are picked up From fb74ea668e9fbc3f630860f0adf7322a6df5f406 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 15:07:21 +0200 Subject: [PATCH 218/256] FIX unify module syntax and tool while modifying --- easybuild/tools/options.py | 1 + test/framework/options.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8198be9c72..3ccbd9e184 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -864,6 +864,7 @@ def _postprocess_config(self): self.options.stop = FETCH_STEP self.options.ignore_osdeps = True self.options.modules_tool = None + self.options.module_syntax = None def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" diff --git a/test/framework/options.py b/test/framework/options.py index 0037e2a7c4..30ddee1731 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2961,9 +2961,11 @@ def test_fetch(self): options = EasyBuildOptions(go_args=['--fetch']) options.postprocess() - self.assertEqual(options.stop, FETCH_STEP) - self.assertEqual(options.modulestool, None) - self.assertTrue(options.ignore_osdeps) + self.assertTrue(options.options.fetch) + self.assertEqual(options.options.stop, 'fetch') + self.assertEqual(options.options.modules_tool, None) + self.assertEqual(options.options.module_syntax, None) + self.assertTrue(options.options.ignore_osdeps) args = ['toy-0.0.eb', '--fetch'] stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) From 45f550ac5496f4878dfa9cc79da53319089c2976 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 29 Mar 2018 13:09:17 +0000 Subject: [PATCH 219/256] Test both 'vectorize': True and 'vectorize': False toolchainopts. --- test/framework/toolchain.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 81677a0c10..69579a3867 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -296,17 +296,19 @@ def test_misc_flags_unique(self): tc.set_options({opt: enable}) tc.prepare() if opt == 'optarch': - flag = '-%s' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)] - elif isinstance(tc.options.options_map[opt], dict): - flag = '-%s' % tc.options.options_map[opt][True] + option = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(tc.arch, tc.cpu_family)] else: - flag = '-%s' % tc.options.options_map[opt] + option = tc.options.options_map[opt] + if not isinstance(option, dict): + option = {True: option} for var in flag_vars: flags = tc.get_variable(var) - if enable: - self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) - else: - self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + for key, value in option.items(): + flag = "-%s" % value + if enable == key: + self.assertTrue(flag in flags, "%s: %s means %s in %s" % (opt, enable, flag, flags)) + else: + self.assertTrue(flag not in flags, "%s: %s means no %s in %s" % (opt, enable, flag, flags)) self.modtool.purge() def test_override_optarch(self): From 8f212fe4154108b3b03331b80614fcff834e3384 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 21 Feb 2017 18:47:47 +0000 Subject: [PATCH 220/256] Add support for gmkl toolchain (GCC + MKL, no MPI) --- easybuild/toolchains/gmkl.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 easybuild/toolchains/gmkl.py diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py new file mode 100644 index 0000000000..ca169c9b25 --- /dev/null +++ b/easybuild/toolchains/gmkl.py @@ -0,0 +1,45 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmkl compiler toolchain (includes GCC, +Intel Math Kernel Library (MKL), and Intel FFTW wrappers). + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Gmkl(GccToolchain, IntelMKL, IntelFFTW): + """ + Compiler toolchain with GCC, Intel Math Kernel Library (MKL) + and Intel FFTW wrappers. + """ + NAME = 'gmkl' + SUBTOOLCHAIN = GccToolchain.NAME From ba82821633784b523e282ff9674cfd11eb12c921 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 25 Apr 2017 14:18:19 +0000 Subject: [PATCH 221/256] Add PGI+MKL toolchain. --- easybuild/toolchains/pmkl.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 easybuild/toolchains/pmkl.py diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py new file mode 100644 index 0000000000..2aeac0290f --- /dev/null +++ b/easybuild/toolchains/pmkl.py @@ -0,0 +1,45 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for pmkl compiler toolchain (includes PGI, +Intel Math Kernel Library (MKL), and Intel FFTW wrappers). + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.pgi import PgiToolchain +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Pmkl(PgiToolchain, IntelMKL, IntelFFTW): + """ + Compiler toolchain with PGI, Intel Math Kernel Library (MKL) + and Intel FFTW wrappers. + """ + NAME = 'pmkl' + SUBTOOLCHAIN = PgiToolchain.NAME From 6969c0b365688badc337a3e0c4646db1f651d248 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 16:04:32 +0200 Subject: [PATCH 222/256] IMPROVE try to mock eb fetch _run_mock_eb seem no to return anything, trying with testing = false --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 30ddee1731..b8117175fa 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2968,7 +2968,7 @@ def test_fetch(self): self.assertTrue(options.options.ignore_osdeps) args = ['toy-0.0.eb', '--fetch'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True) + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) patterns = [ "^== fetching files\.\.\.$" From 42ea197d315cf6205d4ec111fc04fd5e20d51d8a Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 30 May 2017 17:45:21 +0000 Subject: [PATCH 223/256] added gmklc toolchain --- easybuild/toolchains/gmklc.py | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 easybuild/toolchains/gmklc.py diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py new file mode 100644 index 0000000000..6258505b3f --- /dev/null +++ b/easybuild/toolchains/gmklc.py @@ -0,0 +1,46 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmkl compiler toolchain (includes GCC, +Intel Math Kernel Library (MKL), and Intel FFTW wrappers, and CUDA). + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +:author: Maxime Boissonneault (Université Laval, Calcul Québec, Calcul Canada) +""" + +from easybuild.toolchains.gcccuda import GccCUDA +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Gmklc(GccCUDA, IntelMKL, IntelFFTW): + """ + Compiler toolchain with GCC, Intel Math Kernel Library (MKL) + and Intel FFTW wrappers. + """ + NAME = 'gmklc' + SUBTOOLCHAIN = GccCUDA.NAME From 544910752f835c17552577fcfd668c6a7128c41f Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 30 May 2017 17:48:53 +0000 Subject: [PATCH 224/256] fixed non-UTF character --- easybuild/toolchains/gmklc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index 6258505b3f..8759cea06b 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -29,7 +29,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) -:author: Maxime Boissonneault (Université Laval, Calcul Québec, Calcul Canada) +:author: Maxime Boissonneault (Universite Laval, Calcul Quebec, Calcul Canada) """ from easybuild.toolchains.gcccuda import GccCUDA From 817b9ec269c583cce64c96b9409f6b36371fa900 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 30 May 2017 20:54:47 +0000 Subject: [PATCH 225/256] changed description of gmklc --- easybuild/toolchains/gmklc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index 8759cea06b..a5b9a77cec 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -40,7 +40,7 @@ class Gmklc(GccCUDA, IntelMKL, IntelFFTW): """ Compiler toolchain with GCC, Intel Math Kernel Library (MKL) - and Intel FFTW wrappers. + and Intel FFTW wrappers and CUDA. """ NAME = 'gmklc' SUBTOOLCHAIN = GccCUDA.NAME From 574d1209bab20eab2f5e68e5a16d645a0b1f41e4 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 29 Mar 2017 20:09:37 +0000 Subject: [PATCH 226/256] added iimklc toolchain --- easybuild/toolchains/iimklc.py | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 easybuild/toolchains/iimklc.py diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py new file mode 100644 index 0000000000..5b2cd452be --- /dev/null +++ b/easybuild/toolchains/iimklc.py @@ -0,0 +1,46 @@ +## +# Copyright 2012-2016 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iomkl compiler toolchain (includes Intel compilers (icc, ifort), OpenMPI, +CUDA, Intel Math Kernel Library (MKL), and Intel FFTW wrappers. + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (Compute Canada) +:author: Maxime Boissonneault (Compute Canada) +""" + +from easybuild.toolchains.iompic import Iompic +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Iomklc(IccIfortCUDA, IntelMKL, IntelFFTW): + """ + Compiler toolchain with Intel compilers (icc/ifort), + CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. + """ + NAME = 'iimklc' + SUBTOOLCHAIN = IccIfortCUDA.NAME From e99c3345f8e2e64c6381f71faeb3fa0377968c87 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 29 Mar 2017 20:11:56 +0000 Subject: [PATCH 227/256] fixing iimklc --- easybuild/toolchains/iimklc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index 5b2cd452be..6180f793eb 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -32,7 +32,7 @@ :author: Maxime Boissonneault (Compute Canada) """ -from easybuild.toolchains.iompic import Iompic +from easybuild.toolchains.iccifortcuda import IccIfortCUDA from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL From 33ba31d5c2571922392e80b4e6ea1805bda3414d Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 16:39:01 +0200 Subject: [PATCH 228/256] FIX don't amend syntax while parsin options --- easybuild/tools/options.py | 1 - test/framework/options.py | 1 - 2 files changed, 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3ccbd9e184..8198be9c72 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -864,7 +864,6 @@ def _postprocess_config(self): self.options.stop = FETCH_STEP self.options.ignore_osdeps = True self.options.modules_tool = None - self.options.module_syntax = None def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" diff --git a/test/framework/options.py b/test/framework/options.py index b8117175fa..a4c7bca24e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2959,7 +2959,6 @@ def test_stop(self): def test_fetch(self): options = EasyBuildOptions(go_args=['--fetch']) - options.postprocess() self.assertTrue(options.options.fetch) self.assertEqual(options.options.stop, 'fetch') From fd54107378bde21d1d2dfe8805a6c0644c51b92a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 3 Feb 2017 01:14:59 +0000 Subject: [PATCH 229/256] Add iompic (=iompi+CUDA) toolchain. --- easybuild/toolchains/iompic.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 easybuild/toolchains/iompic.py diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py new file mode 100644 index 0000000000..4c231e7fcc --- /dev/null +++ b/easybuild/toolchains/iompic.py @@ -0,0 +1,38 @@ +12;rgb:0000/0000/0000## +# Copyright 2013-2016 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iompic compiler toolchain (includes iccifort, OpenMPI and CUDA). + +:author: Bart Oldeman (Compute Canada) +""" + +from easybuild.toolchains.iccifortcuda import IccIfortCUDA +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class Iompic(IccIfortCUDA, OpenMPI): + """Compiler toolchain with Intel compilers (icc/ifort), OpenMPI and CUDA.""" + NAME = 'iompic' + SUBTOOLCHAIN = IccIfortCUDA.NAME From 5cf9c858145a2f60fd4910cffb08583c25544363 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 3 Feb 2017 01:47:12 +0000 Subject: [PATCH 230/256] Delete garbage. --- easybuild/toolchains/iompic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py index 4c231e7fcc..048238caa2 100644 --- a/easybuild/toolchains/iompic.py +++ b/easybuild/toolchains/iompic.py @@ -1,4 +1,4 @@ -12;rgb:0000/0000/0000## +## # Copyright 2013-2016 Ghent University # # This file is part of EasyBuild, From ea07912f286311a24bbd3db6bf7657a11833cb2d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 3 Feb 2017 02:24:02 +0000 Subject: [PATCH 231/256] Add iomklc toolchain (iomkl + cuda) --- easybuild/toolchains/iomklc.py | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 easybuild/toolchains/iomklc.py diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py new file mode 100644 index 0000000000..fc4e9e7a40 --- /dev/null +++ b/easybuild/toolchains/iomklc.py @@ -0,0 +1,45 @@ +## +# Copyright 2012-2016 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iomkl compiler toolchain (includes Intel compilers (icc, ifort), OpenMPI, +CUDA, Intel Math Kernel Library (MKL), and Intel FFTW wrappers. + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (Compute Canada) +""" + +from easybuild.toolchains.iompic import Iompic +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Iomklc(Iompic, IntelMKL, IntelFFTW): + """ + Compiler toolchain with Intel compilers (icc/ifort), OpenMPI, + CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. + """ + NAME = 'iomklc' + SUBTOOLCHAIN = Iompic.NAME From bb9ddac741e769c8c1799129a033e0902be449b9 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 4 Oct 2017 20:02:48 +0000 Subject: [PATCH 232/256] created a gomklc toolchain --- easybuild/toolchains/gomklc.py | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 easybuild/toolchains/gomklc.py diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py new file mode 100644 index 0000000000..c64f2a933c --- /dev/null +++ b/easybuild/toolchains/gomklc.py @@ -0,0 +1,45 @@ +## +# Copyright 2012-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gimkl compiler toolchain (includes GCC, Open MPI, CUDA +Intel Math Kernel Library (MKL) and Intel FFTW wrappers). + +:author: Stijn De Weirdt (Ghent University) +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (Compute Canada) +:author: Maxime Boissonneault (Compute Canada) +""" + +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.gmkl import Gmkl +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Gomklc(Gompic, Gmkl, IntelMKL, IntelFFTW): + """Compiler toolchain with GCC, Open MPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers and Cuda.""" + NAME = 'gomklc' + SUBTOOLCHAIN = [Gompic.NAME, Gmkl.NAME] + From a06b2536cf3cef877ff3d47012c672e95e0ac766 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 4 Oct 2017 20:19:43 +0000 Subject: [PATCH 233/256] forgot to import gompic --- easybuild/toolchains/gomklc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index c64f2a933c..272529ce87 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -32,7 +32,7 @@ :author: Maxime Boissonneault (Compute Canada) """ -from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.gompic import Gompic from easybuild.toolchains.gmkl import Gmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL From e2c8eae86974423ceb0b9ffde79cb595627400d3 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 29 Mar 2018 14:58:24 +0000 Subject: [PATCH 234/256] Disable multiple subtoolchains in gomklc for now. --- easybuild/toolchains/gomklc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index 272529ce87..6d3448037f 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2017 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,13 +33,12 @@ """ from easybuild.toolchains.gompic import Gompic -from easybuild.toolchains.gmkl import Gmkl from easybuild.toolchains.fft.intelfftw import IntelFFTW from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Gomklc(Gompic, Gmkl, IntelMKL, IntelFFTW): +class Gomklc(Gompic, IntelMKL, IntelFFTW): """Compiler toolchain with GCC, Open MPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers and Cuda.""" NAME = 'gomklc' - SUBTOOLCHAIN = [Gompic.NAME, Gmkl.NAME] + SUBTOOLCHAIN = Gompic.NAME From 9d3d27c67a4ba86c20c766cc2128df175a005d61 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 29 Mar 2018 15:04:48 +0000 Subject: [PATCH 235/256] Fix typos in iimklc toolchain. --- easybuild/toolchains/iimklc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index 6180f793eb..c54abe8441 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2016 Ghent University +# Copyright 2012-2018 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for iomkl compiler toolchain (includes Intel compilers (icc, ifort), OpenMPI, +EasyBuild support for iimklc compiler toolchain (includes Intel compilers (icc, ifort), OpenMPI, CUDA, Intel Math Kernel Library (MKL), and Intel FFTW wrappers. :author: Stijn De Weirdt (Ghent University) @@ -37,7 +37,7 @@ from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Iomklc(IccIfortCUDA, IntelMKL, IntelFFTW): +class Iimklc(IccIfortCUDA, IntelMKL, IntelFFTW): """ Compiler toolchain with Intel compilers (icc/ifort), CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. From 75538f8958be5e76679fbe7ce96c2343381c85c9 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Thu, 29 Mar 2018 17:45:45 +0200 Subject: [PATCH 236/256] CLEAN no syntax modification from previous commit --- test/framework/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index a4c7bca24e..8bfa31f171 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2963,7 +2963,6 @@ def test_fetch(self): self.assertTrue(options.options.fetch) self.assertEqual(options.options.stop, 'fetch') self.assertEqual(options.options.modules_tool, None) - self.assertEqual(options.options.module_syntax, None) self.assertTrue(options.options.ignore_osdeps) args = ['toy-0.0.eb', '--fetch'] From 85aaccf5a78c92de4576ca5520ba5c9574e1a803 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 29 Mar 2018 23:02:29 +0000 Subject: [PATCH 237/256] cuda_toolchains: fix whitespace --- easybuild/toolchains/gomklc.py | 1 - easybuild/toolchains/iimklc.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index 6d3448037f..d3ca69fc0b 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -41,4 +41,3 @@ class Gomklc(Gompic, IntelMKL, IntelFFTW): """Compiler toolchain with GCC, Open MPI, Intel Math Kernel Library (MKL) and Intel FFTW wrappers and Cuda.""" NAME = 'gomklc' SUBTOOLCHAIN = Gompic.NAME - diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index c54abe8441..5bb2cd294b 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -39,7 +39,7 @@ class Iimklc(IccIfortCUDA, IntelMKL, IntelFFTW): """ - Compiler toolchain with Intel compilers (icc/ifort), + Compiler toolchain with Intel compilers (icc/ifort), CUDA, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'iimklc' From b79f81098b9c35897f770da109014a545bccf8ba Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 09:07:20 +0200 Subject: [PATCH 238/256] CLEAN typo --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 8bfa31f171..89e48432d2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2969,8 +2969,8 @@ def test_fetch(self): stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) patterns = [ - "^== fetching files\.\.\.$" - "^== COMPLETED: Installation STOPPED successfully$" + "^== fetching files\.\.\.$", + "^== COMPLETED: Installation STOPPED successfully$", ] for pattern in patterns: regex = re.compile(pattern, re.M) From 7cc7bb61c7e399ed3ce7c9d395bad386d96288b7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 30 Mar 2018 12:18:26 +0200 Subject: [PATCH 239/256] stick to autopep8 1.3.4 when testing with Python 2.6 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa3cf0cd38..6c94e34cd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,8 +47,10 @@ before_install: - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'paramiko<2.4.0'; else pip install paramiko; fi # SQLAlchemy (dep for GC3Pie) 1.2.0 & more recent doesn't work with Python 2.6 - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'SQLAlchemy<1.2.0'; else pip install SQLAlchemy; fi + # autopep8 1.3.4 is last one to support Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'autopep8<1.3.5'; else pip install autopep8; fi # optional Python packages for EasyBuild - - pip install autopep8 GC3Pie pycodestyle python-graph-dot python-hglib PyYAML + - pip install GC3Pie pycodestyle python-graph-dot python-hglib PyYAML # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" - git config --global user.email "travis@travis-ci.org" From 6a1549a72659cd5c9fd80913c017f696099aedcc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 30 Mar 2018 15:51:34 +0200 Subject: [PATCH 240/256] strip off .lua extension when backing up modules to ensure Lmod 6.x doesn't pick up on them --- easybuild/framework/easyblock.py | 12 ++++++++++-- easybuild/tools/filetools.py | 6 +++++- test/framework/filetools.py | 13 ++++++++++++- test/framework/toy_build.py | 10 ++++++---- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a87aa44874..876587324a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1524,12 +1524,20 @@ def check_readiness_step(self): # create backup of existing module file (if requested) if os.path.exists(self.mod_filepath) and build_option('backup_modules'): + # strip off .lua extension to ensure that Lmod ignores backed up module file + # Lmod 7.x ignores any files not ending in .lua + # Lmod 6.x ignores any files that don't have .lua anywhere in the filename + strip_fn = None + if isinstance(self.module_generator, ModuleGeneratorLua): + strip_fn = ModuleGeneratorLua.MODULE_FILE_EXTENSION + # backups of modules in Tcl syntax should be hidden to avoid that they're shown in 'module avail'; # backups of modules in Lua syntax do not need to be hidden: - # since they don't end in .lua (but in .lua.bak_*) Lmod will not pick them up anymore, + # since they don't have .lua in the filename Lmod will not pick them up anymore, # which is better than hiding them (since --show-hidden still reveals them) hidden = isinstance(self.module_generator, ModuleGeneratorTcl) - self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden) + + self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn) print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log) # check if main install needs to be skipped diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c5675c0c23..b2355c2353 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1218,13 +1218,14 @@ def find_backup_name_candidate(src_file): return dst_file -def back_up_file(src_file, backup_extension='bak', hidden=False): +def back_up_file(src_file, backup_extension='bak', hidden=False, strip_fn=None): """ Backs up a file appending a backup extension and timestamp to it (if there is already an existing backup). :param src_file: file to be back up :param backup_extension: extension to use for the backup file (can be empty or None) :param hidden: make backup hidden (leading dot in filename) + :param strip_fn: strip specified trailing substring from filename of backup :return: location of backed up file """ fn_prefix, fn_suffix = '', '' @@ -1234,6 +1235,9 @@ def back_up_file(src_file, backup_extension='bak', hidden=False): fn_suffix = '.%s' % backup_extension src_dir, src_fn = os.path.split(src_file) + if strip_fn: + src_fn = src_fn.rstrip(strip_fn) + backup_fp = find_backup_name_candidate(os.path.join(src_dir, fn_prefix + src_fn + fn_suffix)) copy_file(src_file, backup_fp) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8dbb098160..d63711389f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -567,10 +567,11 @@ def test_back_up_file(self): self.assertEqual(sorted(os.listdir(os.path.dirname(fp))), known_files) # Test simple file backup - ft.back_up_file(fp) + res = ft.back_up_file(fp) test_files = os.listdir(os.path.dirname(fp)) self.assertEqual(len(test_files), 2) new_file = [x for x in test_files if x not in known_files][0] + self.assertTrue(os.path.samefile(res, os.path.join(self.test_prefix, 'sandbox', new_file))) self.assertTrue(new_file.startswith('test.txt.bak_')) first_normal_backup = os.path.join(os.path.dirname(fp), new_file) known_files = os.listdir(os.path.dirname(fp)) @@ -680,6 +681,16 @@ def test_back_up_file(self): self.assertEqual(ft.read_file(os.path.join(os.path.dirname(fp), new_file)), new_txt) self.assertEqual(ft.read_file(fp), new_txt) + # check whether strip_fn works as expected + fp2 = fp + '.lua' + ft.copy_file(fp, fp2) + res = ft.back_up_file(fp2) + self.assertTrue(fp2.endswith('.lua')) + self.assertTrue('.lua' in os.path.basename(res)) + + res = ft.back_up_file(fp2, strip_fn='.lua') + self.assertFalse('.lua' in os.path.basename(res)) + def test_move_logs(self): """Test move_logs function.""" fp = os.path.join(self.test_prefix, 'test.txt') diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d1e081973e..6ef3767c9d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1386,8 +1386,10 @@ def test_backup_modules(self): # Test also with lua syntax if Lmod is available. In particular, that the backup is not hidden if isinstance(self.modtool, Lmod): args = common_args + ['--module-syntax=Lua', '--backup-modules'] - toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps.lua') - toy_mod_dir, toy_mod_fn = os.path.split(toy_mod) + + toy_mod_dir = os.path.join(self.test_installpath, 'modules', 'all', 'toy') + toy_mod_fn = '0.0-deps' + toy_mod = os.path.join(toy_mod_dir, toy_mod_fn + '.lua') self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_mod)) @@ -1406,10 +1408,10 @@ def test_backup_modules(self): toy_mod_backups = glob.glob(os.path.join(toy_mod_dir, toy_mod_fn + '.bak_*')) self.assertEqual(len(toy_mod_backups), 1) first_toy_lua_mod_backup = toy_mod_backups[0] - self.assertTrue('.lua.bak' in os.path.basename(first_toy_lua_mod_backup)) + self.assertTrue('.bak_' in os.path.basename(first_toy_lua_mod_backup)) self.assertFalse(os.path.basename(first_toy_lua_mod_backup).startswith('.')) - toy_mod_bak = ".*/toy/0\.0-deps\.lua\.bak_[0-9]*" + toy_mod_bak = ".*/toy/0\.0-deps\.bak_[0-9]*" regex = re.compile("^== backup of existing module file stored at %s" % toy_mod_bak, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) regex = re.compile("^== comparing module file with backup %s; no differences found$" % toy_mod_bak, re.M) From 707bbe2062ec98d948a4f7aa548cb1f2c4b86952 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 16:52:03 +0200 Subject: [PATCH 241/256] CLEAN addressing #2452 reviews --- easybuild/main.py | 4 ---- easybuild/tools/modules.py | 3 ++- easybuild/tools/options.py | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 10b6f2572c..6b5897e0f5 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -255,10 +255,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) - if options.fetch: - _log.debug("Fetching sources only, implies stop=fetch, moduletool=NoModulesTool, ignore-osdep.") - modtool = NoModulesTool(testing=testing) - if modtool is None: modtool = modules_tool(testing=testing) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 9dce50b668..0d5647d112 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1340,6 +1340,7 @@ def avail_modules_tools(): # filter out legacy Modules class if 'Modules' in class_dict: del class_dict['Modules'] + # NoModulesTool should never be used deliberately, so remove it from the list of available module tools if 'NoModulesTool' in class_dict: del class_dict['NoModulesTool'] return class_dict @@ -1355,7 +1356,7 @@ def modules_tool(mod_paths=None, testing=False): modules_tool_class = avail_modules_tools().get(modules_tool) return modules_tool_class(mod_paths=mod_paths, testing=testing) else: - return None + return NoModulesTool(mod_paths=mod_paths, testing=testing) def reset_module_caches(): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8198be9c72..6d10c6d237 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -255,8 +255,6 @@ def basic_options(self): 'extended-dry-run': ("Print build environment and (expected) build procedure that will be performed", None, 'store_true', False, 'x'), 'extended-dry-run-ignore-errors': ("Ignore errors that occur during dry run", None, 'store_true', True), - 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " - "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module), " "and skipping check for OS dependencies", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), @@ -357,6 +355,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " + "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), From 12498c12b509fb645f33b7e7e10fd22f80bb111f Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 18:35:52 +0200 Subject: [PATCH 242/256] IMPROVE test_fetch: don't take into account the modtool for this one --- test/framework/options.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index d8a3df3182..9e54d643ae 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2978,8 +2978,14 @@ def test_fetch(self): self.assertEqual(options.options.modules_tool, None) self.assertTrue(options.options.ignore_osdeps) + # in this test we want to fake the case were no modules tool are in the system so teak it + original_modtool = self.modtool + self.modtool = None args = ['toy-0.0.eb', '--fetch'] stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) + # back to orinal modules tool so further tests can continue + self.modtool = original_modtool + patterns = [ "^== fetching files\.\.\.$", From bbd1c24c3ce25fb57de0bdd1348998af6f4f3b8c Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 18:59:31 +0200 Subject: [PATCH 243/256] CLEAN style --- test/framework/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9e54d643ae..6ffbf1e9b0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2986,7 +2986,6 @@ def test_fetch(self): # back to orinal modules tool so further tests can continue self.modtool = original_modtool - patterns = [ "^== fetching files\.\.\.$", "^== COMPLETED: Installation STOPPED successfully$", From 690523dbae4bb5739282e2ed19a5e197634abdd8 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 19:06:53 +0200 Subject: [PATCH 244/256] IMPROVE avoid code duplication adressing review --- easybuild/tools/modules.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 0d5647d112..4adf97dc79 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1352,11 +1352,8 @@ def modules_tool(mod_paths=None, testing=False): """ # get_modules_tool might return none (e.g. if config was not initialized yet) modules_tool = get_modules_tool() - if modules_tool is not None: - modules_tool_class = avail_modules_tools().get(modules_tool) - return modules_tool_class(mod_paths=mod_paths, testing=testing) - else: - return NoModulesTool(mod_paths=mod_paths, testing=testing) + modules_tool_class = avail_modules_tools().get(modules_tool, default=NoModulesTool) + return modules_tool_class(mod_paths=mod_paths, testing=testing) def reset_module_caches(): From 540a388925fe05426e0f280cec0d3dab53a5f3f5 Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Fri, 30 Mar 2018 19:18:05 +0200 Subject: [PATCH 245/256] FIX zealous verbosity typo --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 4adf97dc79..e53d684e5f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1352,7 +1352,7 @@ def modules_tool(mod_paths=None, testing=False): """ # get_modules_tool might return none (e.g. if config was not initialized yet) modules_tool = get_modules_tool() - modules_tool_class = avail_modules_tools().get(modules_tool, default=NoModulesTool) + modules_tool_class = avail_modules_tools().get(modules_tool, NoModulesTool) return modules_tool_class(mod_paths=mod_paths, testing=testing) From ed7494dedea2ca54330c4d63c8f0a74088d66e92 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sat, 31 Mar 2018 01:54:30 +0000 Subject: [PATCH 246/256] vectorize: fix [:1] syntax and clarify avoiding double flags. Addresses @boegel's review. --- easybuild/tools/toolchain/compiler.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 66d1f83e6f..d97a0e7e34 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -246,17 +246,18 @@ def _set_compiler_flags(self): (default_opt_level, self.COMPILER_OPT_FLAGS)) # 1st one is the one to use. add default at the end so len is at least 1 - optflags = [self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] + \ - [self.options.option(default_opt_level)][:1] + optflags = ([self.options.option(x) for x in self.COMPILER_OPT_FLAGS if self.options.get(x, False)] + \ + [self.options.option(default_opt_level)])[:1] # only apply if the vectorize toolchainopt is explicitly set # otherwise the individual compiler toolchain file should make sure that # vectorization is disabled for noopt and lowopt, and enabled otherwise. if self.options.get('vectorize') is not None: - vectflags = self.options.option('vectorize')[self.options['vectorize']] - # avoid double use of such flags + vectoptions = self.options.option('vectorize') + vectflags = vectoptions[self.options['vectorize']] + # avoid double use of such flags, or e.g. -fno-tree-vectorize followed by -ftree-vectorize if isinstance(optflags[0], list): - optflags[0] = [flag for flag in optflags[0] if flag not in self.options.option('vectorize').values()] + optflags[0] = [flag for flag in optflags[0] if flag not in vectoptions.values()] optflags.append(vectflags) optarchflags = [] From 631810dd568bad0c6291397afa3bccff5b3c6066 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sat, 31 Mar 2018 02:07:59 +0000 Subject: [PATCH 247/256] Replace *use_depends_on by *depends_on as suggested by @boegel. --- easybuild/framework/easyblock.py | 8 ++++---- easybuild/framework/easyconfig/default.py | 4 ++-- easybuild/tools/config.py | 4 ++-- easybuild/tools/module_generator.py | 8 ++++---- easybuild/tools/options.py | 6 +++--- test/framework/module_generator.py | 8 ++++---- test/framework/options.py | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0085e290bb..dc1c049c13 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -937,7 +937,7 @@ def make_devel_module(self, create_in_builddir=False): # capture all the EBDEVEL vars # these should be all the dependencies and we should load them recursive_unload = self.cfg['recursive_module_unload'] - use_depends_on = self.cfg['module_use_depends_on'] + depends_on = self.cfg['module_depends_on'] for key in os.environ: # legacy support if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): @@ -946,7 +946,7 @@ def make_devel_module(self, create_in_builddir=False): if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload, - use_depends_on=use_depends_on) + depends_on=depends_on) load_lines.append(load_statement) elif key.startswith('SOFTDEVEL'): self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') @@ -1045,14 +1045,14 @@ def make_module_dep(self, unload_info=None): # include load statements for retained dependencies recursive_unload = self.cfg['recursive_module_unload'] - use_depends_on = self.cfg['module_use_depends_on'] + depends_on = self.cfg['module_depends_on'] loads = [] for dep in deps: unload_modules = [] if dep in unload_info: unload_modules.append(unload_info[dep]) loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload, - use_depends_on=use_depends_on, + depends_on=depends_on, unload_modules=unload_modules)) # Force unloading any other modules diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 1e3e078ce2..7f68136361 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -171,8 +171,8 @@ 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], - 'module_use_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' - '(implies recursive unloading of modules).', MODULES], + 'module_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' + '(implies recursive unloading of modules).', MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], # MODULES documentation easyconfig parameters diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0d76e1f4bf..28b471a64f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -236,7 +236,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): False: [ 'dry_run', 'recursive_mod_unload', - 'mod_use_depends_on', + 'mod_depends_on', 'retain_all_deps', 'silent', 'try_to_generate', @@ -387,7 +387,7 @@ def init_build_options(build_options=None, cmdline_options=None): 'check_osdeps': not cmdline_options.ignore_osdeps, 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, 'recursive_mod_unload': cmdline_options.recursive_module_unload, - 'mod_use_depends_on': cmdline_options.module_use_depends_on, + 'mod_depends_on': cmdline_options.module_depends_on, 'retain_all_deps': retain_all_deps, 'validate': not cmdline_options.force, 'valid_module_classes': module_classes(), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 925eb6a6fe..d9b5c70469 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -495,7 +495,7 @@ def getenv_cmd(self, envvar): """ return '$env(%s)' % envvar - def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -508,7 +508,7 @@ def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, un body.extend([self.unload_module(m).strip() for m in unload_modules]) load_template = self.LOAD_TEMPLATE # Lmod 7.6.1+ supports depends-on which does this most nicely: - if build_option('mod_use_depends_on') or use_depends_on: + if build_option('mod_depends_on') or depends_on: if not modules_tool().supports_depends_on: raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON @@ -793,7 +793,7 @@ def getenv_cmd(self, envvar): """ return 'os.getenv("%s")' % envvar - def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -807,7 +807,7 @@ def load_module(self, mod_name, recursive_unload=False, use_depends_on=False, un load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: - if build_option('mod_use_depends_on') or use_depends_on: + if build_option('mod_depends_on') or depends_on: if not modules_tool().supports_depends_on: raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ea62c5296e..b696ae5b44 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -453,12 +453,12 @@ def config_options(self): # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), + 'module-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules " + "(implies recursive unloading of modules).", + None, 'store_true', False), 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), - 'module-use-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules " - "(implies recursive unloading of modules).", - None, 'store_true', False), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), 'strlist', 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index d8c62aa722..c8fc611b46 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -266,8 +266,8 @@ def test_load(self): "depends-on mod_name", '', ]) - self.assertEqual(expected, self.modgen.load_module("mod_name", use_depends_on=True)) - init_config(build_options={'mod_use_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) + init_config(build_options={'mod_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) else: # default: guarded module load (which implies no recursive unloading) @@ -301,8 +301,8 @@ def test_load(self): 'depends_on("mod_name")', '', ]) - self.assertEqual(expected, self.modgen.load_module("mod_name", use_depends_on=True)) - init_config(build_options={'mod_use_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) + init_config(build_options={'mod_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) def test_unload(self): diff --git a/test/framework/options.py b/test/framework/options.py index 0420db0812..6847aff8b7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1158,7 +1158,7 @@ def test_recursive_module_unload(self): # check log message with --skip for existing module lastargs = ['--recursive-module-unload'] if self.modtool.supports_depends_on: - lastargs.append('--module-use-depends-on') + lastargs.append('--module-depends-on') for lastarg in lastargs: args = [ eb_file, From 4b1c65e759800d656be5c15a492f815d98c19ba1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sat, 31 Mar 2018 02:15:50 +0000 Subject: [PATCH 248/256] Undo wrapping for clarity. --- easybuild/tools/module_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index d9b5c70469..315dca46c8 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -514,8 +514,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) - if (build_option('recursive_mod_unload') or recursive_unload or - load_template == self.LOAD_TEMPLATE_DEPENDS_ON): + if build_option('recursive_mod_unload') or recursive_unload or load_template == self.LOAD_TEMPLATE_DEPENDS_ON: # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading; # when "module unload" is called on the module in which the dependency "module load" is present, # it will get translated to "module unload" From 84094e87c0295513eba61f06cee2edd8b19ad679 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Sat, 31 Mar 2018 02:57:57 +0000 Subject: [PATCH 249/256] Test if module-depends-on gives an error with unsupported module tools. --- easybuild/tools/module_generator.py | 2 +- test/framework/module_generator.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 315dca46c8..375fe7e160 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -510,7 +510,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload # Lmod 7.6.1+ supports depends-on which does this most nicely: if build_option('mod_depends_on') or depends_on: if not modules_tool().supports_depends_on: - raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") + raise EasyBuildError("depends-on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index c8fc611b46..1220946d62 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -269,6 +269,11 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) init_config(build_options={'mod_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + else: + expected = "depends-on statements in generated module are not supported by modules tool" + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name") else: # default: guarded module load (which implies no recursive unloading) expected = '\n'.join([ @@ -304,6 +309,11 @@ def test_load(self): self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) init_config(build_options={'mod_depends_on': 'True'}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + else: + expected = "depends_on statements in generated module are not supported by modules tool" + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name") def test_unload(self): """Test unload part in generated module file.""" From 23cbc4f6e90a1358186573a5a8dfda314fe47c5b Mon Sep 17 00:00:00 2001 From: Victor Cameo Date: Tue, 3 Apr 2018 09:11:53 +0200 Subject: [PATCH 250/256] CLEAN useless code --- easybuild/main.py | 2 +- test/framework/options.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 6b5897e0f5..02345fb867 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -62,7 +62,7 @@ from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook -from easybuild.tools.modules import modules_tool, NoModulesTool +from easybuild.tools.modules import modules_tool from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support diff --git a/test/framework/options.py b/test/framework/options.py index 6ffbf1e9b0..b8452702e4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2979,12 +2979,9 @@ def test_fetch(self): self.assertTrue(options.options.ignore_osdeps) # in this test we want to fake the case were no modules tool are in the system so teak it - original_modtool = self.modtool self.modtool = None args = ['toy-0.0.eb', '--fetch'] stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) - # back to orinal modules tool so further tests can continue - self.modtool = original_modtool patterns = [ "^== fetching files\.\.\.$", From e5fa2605e2e1c75af68f23586f00cff4a84d1b4f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Apr 2018 11:53:24 +0200 Subject: [PATCH 251/256] check for modulecmd.tcl before modulecmd in bootstrap script to discriminate between Modules 4.1.x vs 3.2.10 --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c94e34cd2..1ba4971119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,7 +99,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180201.01 9d6278409b3d9ecf034bd28384da25c14ebfe773606d4fa800e51ed11667e851" + - EB_BOOTSTRAP_EXPECTED="20180408.01 60b1f1900fb6eb96cdb1206abfd25ab54193ebe2fb71c8402975e843d1aec5ce" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 339c18c71a..ebca798d51 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -53,7 +53,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180201.01' +EB_BOOTSTRAP_VERSION = '20180408.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -259,8 +259,8 @@ def is_modulecmd_tcl_modulestcl(): # order matters, which is why we don't use a dict known_module_commands = [ ('lmod', 'Lmod'), - ('modulecmd', 'EnvironmentModulesC'), ('modulecmd.tcl', 'EnvironmentModules'), + ('modulecmd', 'EnvironmentModulesC'), ] out = os.path.join(tmpdir, 'module_command.out') modtool = None From 23f4c28381fbf722cc3eccba75f037829dd46826 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Apr 2018 16:20:50 +0200 Subject: [PATCH 252/256] fix derive_alt_pypi_url after PyPI switching to sha256 in package URLs + fix broken test for pypi_source_urls --- easybuild/tools/filetools.py | 11 +++++++---- test/framework/filetools.py | 10 +++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b2355c2353..c308f44d80 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -423,16 +423,19 @@ def derive_alt_pypi_url(url): cand_urls = pypi_source_urls(pkg_name) - regex = re.compile('.*/%s#md5=[a-f0-9]{32}$' % pkg_source.replace('.', '\\.'), re.M) + # md5 for old PyPI, sha256 for new PyPi (Warehouse) + regex_md5 = re.compile('.*/%s#md5=[a-f0-9]{32}$' % pkg_source.replace('.', '\\.'), re.M) + regex_sha256 = re.compile('.*/%s#sha256=[a-f0-9]{64}$' % pkg_source.replace('.', '\\.'), re.M) for cand_url in cand_urls: - res = regex.match(cand_url) + res = regex_sha256.match(cand_url) or regex_md5.match(cand_url) if res: # e.g.: https://pypi.python.org/packages////easybuild-.tar.gz#md5= - alt_pypi_url = res.group(0).split('#md5')[0] + alt_pypi_url = res.group(0).split('#sha256')[0].split('#md5')[0] break if not alt_pypi_url: - _log.debug("Failed to extract hash using pattern '%s' from list of URLs: %s", regex.pattern, cand_urls) + _log.debug("Failed to extract hash using pattern '%s' or '%s' from list of URLs: %s", + regex_sha256.pattern, regex_md5.pattern, cand_urls) return alt_pypi_url diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d63711389f..e25d16cb1e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1114,10 +1114,14 @@ def test_pypi_source_urls(self): eb340_url += 'easybuild-3.4.0.tar.gz#md5=267a056a77a8f77fccfbf56354364045' self.assertTrue(eb340_url, res) pattern = '^https://pypi.python.org/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/' - pattern += 'easybuild-[0-9rc.]+.tar.gz#md5=[a-f0-9]{32}$' - regex = re.compile(pattern) + pattern_md5 = pattern + 'easybuild-[0-9rc.]+.tar.gz#md5=[a-f0-9]{32}$' + pattern_sha256 = pattern + 'easybuild-[0-9rc.]+.tar.gz#sha256=[a-f0-9]{64}$' + regex_md5 = re.compile(pattern_md5) + regex_sha256 = re.compile(pattern_sha256) for url in res: - self.assertTrue(regex.match(url), "Pattern '%s' matches for '%s'" % (regex.pattern, url)) + error_msg = "Pattern '%s' or '%s' matches for '%s'" % (regex_md5.pattern, regex_sha256.pattern, url) + self.assertTrue(regex_md5.match(url) or regex_sha256.match(url), error_msg) + # more than 50 releases at time of writing test, which always stay there self.assertTrue(len(res) > 50) From 227c5783c34bcbad73907f3ba3c8cfe24d5e636e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Apr 2018 17:54:21 +0200 Subject: [PATCH 253/256] use a single regex in derive_alt_pypi_url --- easybuild/tools/filetools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c308f44d80..4720b0e98b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -424,18 +424,16 @@ def derive_alt_pypi_url(url): cand_urls = pypi_source_urls(pkg_name) # md5 for old PyPI, sha256 for new PyPi (Warehouse) - regex_md5 = re.compile('.*/%s#md5=[a-f0-9]{32}$' % pkg_source.replace('.', '\\.'), re.M) - regex_sha256 = re.compile('.*/%s#sha256=[a-f0-9]{64}$' % pkg_source.replace('.', '\\.'), re.M) + regex = re.compile('.*/%s(?:#md5=[a-f0-9]{32}|#sha256=[a-f0-9]{64})$' % pkg_source.replace('.', '\\.'), re.M) for cand_url in cand_urls: - res = regex_sha256.match(cand_url) or regex_md5.match(cand_url) + res = regex.match(cand_url) if res: # e.g.: https://pypi.python.org/packages////easybuild-.tar.gz#md5= alt_pypi_url = res.group(0).split('#sha256')[0].split('#md5')[0] break if not alt_pypi_url: - _log.debug("Failed to extract hash using pattern '%s' or '%s' from list of URLs: %s", - regex_sha256.pattern, regex_md5.pattern, cand_urls) + _log.debug("Failed to extract hash using pattern '%s' or '%s' from list of URLs: %s", regex.pattern, cand_urls) return alt_pypi_url From 96648314b7175d47a0ed348dc38c681eb55f13ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Apr 2018 18:40:15 +0200 Subject: [PATCH 254/256] fix issue with deriving PyPI downloads URLs in bootstrap script --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 32 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ba4971119..1d67daf37d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,7 +99,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180408.01 60b1f1900fb6eb96cdb1206abfd25ab54193ebe2fb71c8402975e843d1aec5ce" + - EB_BOOTSTRAP_EXPECTED="20180412.01 06267c2423de76d49055360e618399fdaebc2e5f3e69e50fabfb6a433ab9744b" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index ebca798d51..082b9dbc7e 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -53,7 +53,7 @@ from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180408.01' +EB_BOOTSTRAP_VERSION = '20180412.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -659,14 +659,36 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): 'preinstallopts': preinstallopts, }) - # create easyconfig file - ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) - handle = open(ebfile, 'w') + # need to initialise build options before pypi_source_urls function can be used + from test.framework.utilities import init_config + init_config() + + # determine PyPI URLs for individual packages + from easybuild.tools.filetools import pypi_source_urls + pkg_urls = [] + for pkg in EASYBUILD_PACKAGES: + # format of pkg entries in templates: "''," + pkg_filename = templates[pkg][1:-2] + + # the lines below implement a simplified version of the 'derive_alt_pypi_url' function, + # which we can't leverage yet because of transitional changes in PyPI (#md5= -> #sha256=) + + # figure out relevant URL for source tarball of this pkg version + pkg_url = [x for x in pypi_source_urls(pkg) if '/%s#' % pkg_filename in x][0] + # strip of source tarball filename (and checksum) at the end + pkg_url = re.sub('/[^/]+$', '', pkg_url) + + pkg_urls.append(pkg_url) + templates.update({ - 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), + 'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) + + # create easyconfig file + ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) + handle = open(ebfile, 'w') handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() From 9f488c2e37e80d0927e0f0c2ed1c7b800d75f608 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Apr 2018 18:47:21 +0200 Subject: [PATCH 255/256] fix faulty debug log statement in derive_alt_pypi_url --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4720b0e98b..c8a128bf5a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -433,7 +433,7 @@ def derive_alt_pypi_url(url): break if not alt_pypi_url: - _log.debug("Failed to extract hash using pattern '%s' or '%s' from list of URLs: %s", regex.pattern, cand_urls) + _log.debug("Failed to extract hash using pattern '%s' from list of URLs: %s", regex.pattern, cand_urls) return alt_pypi_url From e5b73b0b3ef8b9011a1344173c3e9eac3ea69f61 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Apr 2018 19:33:50 +0200 Subject: [PATCH 256/256] don't use pypi_source_urls in bootstrap script (since that requires initialised build options), implement lightweight version instead --- .travis.yml | 2 +- easybuild/scripts/bootstrap_eb.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d67daf37d..96881e5e9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,7 +99,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20180412.01 06267c2423de76d49055360e618399fdaebc2e5f3e69e50fabfb6a433ab9744b" + - EB_BOOTSTRAP_EXPECTED="20180412.02 d05b0922f83ba7febfeb1dcfca4a854be93cbfefa6cdedcc87cc8c3835aa3fa3" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 082b9dbc7e..a56c47978d 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -49,11 +49,12 @@ import sys import tempfile import traceback +import urllib2 from distutils.version import LooseVersion from hashlib import md5 -EB_BOOTSTRAP_VERSION = '20180412.01' +EB_BOOTSTRAP_VERSION = '20180412.02' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -659,25 +660,25 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): 'preinstallopts': preinstallopts, }) - # need to initialise build options before pypi_source_urls function can be used - from test.framework.utilities import init_config - init_config() - # determine PyPI URLs for individual packages - from easybuild.tools.filetools import pypi_source_urls pkg_urls = [] for pkg in EASYBUILD_PACKAGES: # format of pkg entries in templates: "''," pkg_filename = templates[pkg][1:-2] - # the lines below implement a simplified version of the 'derive_alt_pypi_url' function, - # which we can't leverage yet because of transitional changes in PyPI (#md5= -> #sha256=) + # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions, + # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=) - # figure out relevant URL for source tarball of this pkg version - pkg_url = [x for x in pypi_source_urls(pkg) if '/%s#' % pkg_filename in x][0] - # strip of source tarball filename (and checksum) at the end - pkg_url = re.sub('/[^/]+$', '', pkg_url) + # determine download URL via PyPI's 'simple' API + pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read() + pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename) + res = pkg_url_part_regex.search(pkg_simple) + if res: + pkg_url_part = res.group(1) + else: + error("Failed to determine PyPI package URL for %s: %s\n" % (pkg, pkg_simple)) + pkg_url = 'https://pypi.python.org/' + pkg_url_part pkg_urls.append(pkg_url) templates.update({