Merge pull request #3 from boegel/add-ieee-754-conformance-as-option

sync with develop & resolve conflicts

easybuild/framework/easyconfig/easyconfig.py

@@ -79,6 +79,9 @@
 # set of configure/build/install options that can be provided as lists for an iterated build
 ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts']
 
+# name of easyconfigs archive subdirectory
+EASYCONFIGS_ARCHIVE_DIR = '__archive__'
+
 
 try:
     import autopep8
@@ -1334,6 +1337,10 @@ def robot_find_easyconfig(name, version):
     elif not isinstance(paths, (list, tuple)):
         paths = [paths]
 
+    # if we should also consider archived easyconfigs, duplicate paths list with archived equivalents
+    if build_option('consider_archived_easyconfigs'):
+        paths = paths + [os.path.join(p, EASYCONFIGS_ARCHIVE_DIR) for p in paths]
+
     res = None
     for path in paths:
         easyconfigs_paths = create_paths(path, name, version)

easybuild/framework/easyconfig/tools.py

@@ -46,7 +46,7 @@
 from vsc.utils import fancylogger
 
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
-from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig
+from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig
 from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig
 from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars
 from easybuild.tools.build_log import EasyBuildError, print_msg
@@ -344,6 +344,10 @@ def det_easyconfig_paths(orig_paths):
                 # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
                 dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')]
 
+                # ignore archived easyconfigs, unless specified otherwise
+                if not build_option('consider_archived_easyconfigs'):
+                    dirnames[:] = [d for d in dirnames if d != EASYCONFIGS_ARCHIVE_DIR]
+
             # stop os.walk insanity as soon as we have all we need (outer loop)
             if not ecs_to_find:
                 break

easybuild/main.py

@@ -178,10 +178,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # set by option parsers via set_tmpdir
     eb_tmpdir = tempfile.gettempdir()
 
+    search_query = options.search or options.search_filename or options.search_short
+
     # initialise logging for main
     global _log
     _log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
-                                 silent=(testing or options.terse), colorize=options.color)
+                                 silent=(testing or options.terse or search_query), colorize=options.color)
 
     # disallow running EasyBuild as root
     if os.getuid() == 0:
@@ -199,8 +201,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # software name/version, toolchain name/version, extra patches, ...
     (try_to_generate, build_specs) = process_software_build_specs(options)
 
-    search_query = options.search or options.search_filename or options.search_short
-
     # determine robot path
     # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
     tweaked_ecs = try_to_generate and build_specs

easybuild/scripts/bootstrap_eb.py

@@ -53,7 +53,7 @@
 from hashlib import md5
 
 
-EB_BOOTSTRAP_VERSION = '20161005.01'
+EB_BOOTSTRAP_VERSION = '20161011.01'
 
 # argparse preferrred, optparse deprecated >=2.7
 HAVE_ARGPARSE = False
@@ -66,7 +66,8 @@
 PYPI_SOURCE_URL = 'https://pypi.python.org/packages/source'
 
 VSC_BASE = 'vsc-base'
-EASYBUILD_PACKAGES = [VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs']
+VSC_INSTALL = 'vsc-install'
+EASYBUILD_PACKAGES = [VSC_INSTALL, VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs']
 
 # set print_debug to True for detailed progress info
 print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False)
@@ -326,10 +327,11 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir):
             if len(pkg_tarball_paths) > 1:
                 error("Multiple tarballs found for %s: %s" % (pkg, pkg_tarball_paths))
             elif len(pkg_tarball_paths) == 0:
-                if pkg != VSC_BASE:
+                if pkg not in [VSC_BASE, VSC_INSTALL]:
                     # vsc-base package is not strictly required
                     # it's only a dependency since EasyBuild v2.0;
-                    # with EasyBuild v2.0, it will be pulled in from PyPI when installing easybuild-framework
+                    # with EasyBuild v2.0, it will be pulled in from PyPI when installing easybuild-framework;
+                    # vsc-install is an optional dependency, only required to run unit tests
                     error("Missing source tarball: %s" % pkg_tarball_glob)
             else:
                 info("Found %s for %s package" % (pkg_tarball_paths[0], pkg))
@@ -414,7 +416,7 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir):
 
         pkg_egg_dir = find_egg_dir_for(targetdir_stage1, pkg)
         if pkg_egg_dir is None:
-            if pkg == VSC_BASE:
+            if pkg in [VSC_BASE, VSC_INSTALL]:
                 # vsc-base is optional in older EasyBuild versions
                 continue
 
@@ -515,7 +517,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
     handle = open(ebfile, 'w')
     templates.update({
         'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
-        'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
+        'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
         'pythonpath': distribute_egg_dir,
     })
     handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)

easybuild/toolchains/compiler/craype.py

@@ -137,7 +137,6 @@ class CrayPEGCC(CrayPECompiler):
     def __init__(self, *args, **kwargs):
         """CrayPEGCC constructor."""
         super(CrayPEGCC, self).__init__(*args, **kwargs)
-        self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = 'fopenmp'
         for precflag in self.COMPILER_PREC_FLAGS:
             self.COMPILER_UNIQUE_OPTION_MAP[precflag] = Gcc.COMPILER_UNIQUE_OPTION_MAP[precflag]
 
@@ -150,7 +149,6 @@ class CrayPEIntel(CrayPECompiler):
     def __init__(self, *args, **kwargs):
         """CrayPEIntel constructor."""
         super(CrayPEIntel, self).__init__(*args, **kwargs)
-        self.COMPILER_UNIQUE_OPTION_MAP['openmp'] = 'fopenmp'
         for precflag in self.COMPILER_PREC_FLAGS:
             self.COMPILER_UNIQUE_OPTION_MAP[precflag] = IntelIccIfort.COMPILER_UNIQUE_OPTION_MAP[precflag]
 

easybuild/toolchains/compiler/cuda.py

@@ -92,8 +92,8 @@ def _set_compiler_flags(self):
             'Xcompiler="%s"' % str(self.variables['CXXFLAGS']),
             'Xlinker="%s %s"' % (str(self.variables['LDFLAGS']), str(self.variables['LIBS'])),
         ]
-        self.variables.nappend('CUDA_CFLAGS', cuda_flags)
-        self.variables.nappend('CUDA_CXXFLAGS', cuda_flags)
+        self.variables.nextend('CUDA_CFLAGS', cuda_flags)
+        self.variables.nextend('CUDA_CXXFLAGS', cuda_flags)
 
         # add gencode compiler flags to list of flags for compiler variables
         for gencode_val in self.options.get('cuda_gencode', []):

easybuild/toolchains/compiler/gcc.py

@@ -56,7 +56,6 @@ class Gcc(Compiler):
         'f2c': 'ff2c',
         'loop': ['ftree-switch-conversion', 'floop-interchange', 'floop-strip-mine', 'floop-block'],
         'lto': 'flto',
-        'openmp': 'fopenmp',
         'ieee': ['mieee-fp', 'fno-trapping-math'],
         'strict': ['mieee-fp', 'mno-recip'],
         'precise':['mno-recip'],

easybuild/toolchains/compiler/inteliccifort.py

@@ -58,7 +58,6 @@ class IntelIccIfort(Compiler):
         'i8': 'i8',
         'r8': 'r8',
         'optarch': 'xHost',
-        'openmp': 'fopenmp',  # both -qopenmp/-fopenmp are valid for enabling OpenMP (-openmp is deprecated)
         'ieee': 'fltconsistency',
         'strict': ['fp-speculation=strict', 'fp-model strict'],
         'precise': ['fp-model precise'],

easybuild/tools/config.py

@@ -132,6 +132,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     False: [
         'add_dummy_to_minimal_toolchains',
         'allow_modules_tool_mismatch',
+        'consider_archived_easyconfigs',
         'debug',
         'debug_lmod',
         'dump_autopep8',

easybuild/tools/filetools.py

@@ -384,7 +384,15 @@ def find_easyconfigs(path, ignore_dirs=None):
 
 def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False):
     """
-    Search for a particular file (only prints)
+    Search for files using in specified paths using specified search query (regular expression)
+
+    :param paths: list of paths to search in
+    :param query: search query to use (regular expression); will be used case-insensitive
+    :param short: figure out common prefix of hits, use variable to factor it out
+    :param ignore_dirs: list of directories to ignore (default: ['.git', '.svn'])
+    :param silent: whether or not to remain silent (don't print anything)
+    :param filename_only: only return filenames, not file paths
+    :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing
     """
     if ignore_dirs is None:
         ignore_dirs = ['.git', '.svn']
@@ -395,50 +403,43 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
     # compile regex, case-insensitive
     query = re.compile(query, re.I)
 
-    var_lines = []
-    hit_lines = []
+    var_defs = []
+    hits = []
     var_index = 1
     var = None
     for path in paths:
-        hits = []
-        hit_in_path = False
+        path_hits = []
         if not terse:
             print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
 
         for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
             for filename in filenames:
                 if query.search(filename):
-                    if not hit_in_path:
+                    if not path_hits:
                         var = "CFGS%d" % var_index
                         var_index += 1
-                        hit_in_path = True
                     if filename_only:
-                        hits.append(filename)
+                        path_hits.append(filename)
                     else:
-                        hits.append(os.path.join(dirpath, filename))
+                        path_hits.append(os.path.join(dirpath, filename))
 
             # do not consider (certain) hidden directories
             # note: we still need to consider e.g., .local !
             # replace list elements using [:], so os.walk doesn't process deleted directories
             # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders
             dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
 
-        hits = sorted(hits)
+        path_hits = sorted(path_hits)
 
-        if hits and not terse:
-            common_prefix = det_common_path_prefix(hits)
-            if short and common_prefix is not None and len(common_prefix) > len(var) * 2:
-                var_lines.append("%s=%s" % (var, common_prefix))
-                hit_lines.extend([" * %s" % os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in hits])
+        if path_hits:
+            common_prefix = det_common_path_prefix(path_hits)
+            if not terse and short and common_prefix is not None and len(common_prefix) > len(var) * 2:
+                var_defs.append((var, common_prefix))
+                hits.extend([os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in path_hits])
             else:
-                hit_lines.extend([" * %s" % fn for fn in hits])
+                hits.extend(path_hits)
 
-    if terse:
-        for line in hits:
-            print(line)
-    else:
-        for line in var_lines + hit_lines:
-            print_msg(line, log=_log, silent=silent, prefix=False)
+    return var_defs, hits
 
 
 def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM):

easybuild/tools/options.py

@@ -310,6 +310,7 @@ def override_options(self):
             'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True),
             'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize,
                       {'metavar':'WHEN'}),
+            'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False),
             'debug-lmod': ("Run Lmod modules tool commands in debug module", None, 'store_true', False),
             'default-opt-level': ("Specify default optimisation level", 'choice', 'store', DEFAULT_OPT_LEVEL,
                                   Compiler.COMPILER_OPT_FLAGS),

easybuild/tools/robot.py

@@ -39,6 +39,7 @@
 from vsc.utils import fancylogger
 from vsc.utils.missing import nub
 
+from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR
 from easybuild.framework.easyconfig.easyconfig import ActiveMNS, process_easyconfig, robot_find_easyconfig
 from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
 from easybuild.tools.build_log import EasyBuildError
@@ -353,12 +354,50 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False):
 
 def search_easyconfigs(query, short=False, filename_only=False, terse=False):
     """Search for easyconfigs, if a query is provided."""
-    robot_path = build_option('robot_path')
-    if robot_path:
-        search_path = robot_path
-    else:
+    search_path = build_option('robot_path')
+    if not search_path:
         search_path = [os.getcwd()]
+
     ignore_dirs = build_option('ignore_dirs')
-    silent = build_option('silent')
-    search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent, filename_only=filename_only,
-                terse=terse)
+
+    # note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs
+    var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse,
+                                  silent=True, filename_only=False)
+
+     # filter out archived easyconfigs, these are handled separately
+    hits, archived_hits = [], []
+    for hit in _hits:
+        if EASYCONFIGS_ARCHIVE_DIR in hit.split(os.path.sep):
+            archived_hits.append(hit)
+        else:
+            hits.append(hit)
+
+    # check whether only filenames should be printed
+    if filename_only:
+        hits = [os.path.basename(hit) for hit in hits]
+        archived_hits = [os.path.basename(hit) for hit in archived_hits]
+
+    # prepare output format
+    if terse:
+        lines, tmpl = [], '%s'
+    else:
+        lines = ['%s=%s' % var_def for var_def in var_defs]
+        tmpl = ' * %s'
+
+    # non-archived hits are shown first
+    lines.extend(tmpl % hit for hit in hits)
+
+    # also take into account archived hits
+    if archived_hits:
+        if build_option('consider_archived_easyconfigs'):
+            if not terse:
+                lines.extend(['', "Matching archived easyconfigs:", ''])
+            lines.extend(tmpl % hit for hit in archived_hits)
+        elif not terse:
+            cnt = len(archived_hits)
+            lines.extend([
+                '',
+                "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt,
+            ])
+
+    print '\n'.join(lines)

easybuild/tools/toolchain/compiler.py

@@ -89,18 +89,19 @@ class Compiler(Toolchain):
 
     COMPILER_UNIQUE_OPTION_MAP = None
     COMPILER_SHARED_OPTION_MAP = {
-        'pic': 'fPIC',
-        'verbose': 'v',
-        'debug': 'g',
-        'unroll': 'unroll',
-        'static': 'static',
-        'shared': 'shared',
-        'noopt': 'O0',
-        'lowopt': 'O1',
         DEFAULT_OPT_LEVEL: 'O2',
-        'opt': 'O3',
         '32bit' : 'm32',
         'cstd': 'std=%(value)s',
+        'debug': 'g',
+        'lowopt': 'O1',
+        'noopt': 'O0',
+        'openmp': 'fopenmp',
+        'opt': 'O3',
+        'pic': 'fPIC',
+        'shared': 'shared',
+        'static': 'static',
+        'unroll': 'unroll',
+        'verbose': 'v',
     }
 
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None
@@ -260,13 +261,13 @@ def _set_compiler_flags(self):
 
         # precflags last
         for var in ['CFLAGS', 'CXXFLAGS']:
-            self.variables.nappend(var, flags)
-            self.variables.nappend(var, cflags)
+            self.variables.nextend(var, flags)
+            self.variables.nextend(var, cflags)
             self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
 
         for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']:
-            self.variables.nappend(var, flags)
-            self.variables.nappend(var, fflags)
+            self.variables.nextend(var, flags)
+            self.variables.nextend(var, fflags)
             self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS')
 
     def _set_optimal_architecture(self):

test/framework/easyconfigs/test_ecs/__archive__/i/ictce/ictce-3.2.2.u3.eb

@@ -0,0 +1,19 @@
+easyblock = "Toolchain"
+
+name = 'ictce'
+version = '3.2.2.u3'
+
+homepage = 'http://software.intel.com/en-us/intel-cluster-toolchain-compiler/'
+description = """Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL."""
+
+toolchain = {'name': 'dummy', 'version': 'dummy'}
+
+# fake/empty ictce/3.2.2.u3 toolchain, just for testing purposes
+dependencies = [
+#    ('icc', '11.1.073'),
+#    ('ifort', '11.1.073'),
+#    ('impi', '4.0.0.028', '', ('iccifort', compver)),
+#    ('imkl', '10.2.6.038', '', ('iimpi', version)),
+]
+
+moduleclass = 'toolchain'