diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index b78919bba1..d5b676bde1 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -53,38 +53,38 @@ class Clang(Compiler): 'loop-vectorize': ['fvectorize'], 'basic-block-vectorize': ['fslp-vectorize'], 'optarch':'march=native', - - # Clang's options do not map well onto these precision modes. The flags enable and disable certain classes of - # optimizations. - # - # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the - # ISO C and C++ language standard by possibly changing computation result. - # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division. - # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant. - # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs. - # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs. - # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results - # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities) - # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros, - # -freciprocal-math). - # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro - # __FAST_MATH__. - # - # Using -fno-fast-math is equivalent to disabling all individual optimizations, see - # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following) - # - # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details - # flags for strict and precise for robustness against future changes. - 'strict': ['fno-fast-math'], - 'precise': ['fno-unsafe-math-optimizations'], - 'defaultprec': [], - 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], - 'veryloose': ['ffast-math'], + # Clang's options do not map well onto these precision modes. The flags enable and disable certain classes of + # optimizations. + # + # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the + # ISO C and C++ language standard by possibly changing computation result. + # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division. + # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant. + # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs. + # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs. + # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results + # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities) + # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros, + # -freciprocal-math). + # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro + # __FAST_MATH__. + # + # Using -fno-fast-math is equivalent to disabling all individual optimizations, see + # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following) + # + # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details + # flags for strict and precise for robustness against future changes. + 'strict': ['fno-fast-math'], + 'precise': ['fno-unsafe-math-optimizations'], + 'defaultprec': [], + 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], + 'veryloose': ['ffast-math'], } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { systemtools.INTEL : 'march=native', - systemtools.AMD : 'march=native' + systemtools.AMD : 'march=native', + systemtools.POWER: 'mcpu=native', # no support for march=native on POWER } COMPILER_CC = 'clang' diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index e47a9c4778..3ec1888438 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -43,31 +43,30 @@ class Gcc(Compiler): COMPILER_FAMILY = TC_CONSTANT_GCC COMPILER_UNIQUE_OPTS = { - 'loop': (False, "Automatic loop parallellisation"), - 'f2c': (False, "Generate code compatible with f2c and f77"), - 'lto':(False, "Enable Link Time Optimization"), - } + 'loop': (False, "Automatic loop parallellisation"), + 'f2c': (False, "Generate code compatible with f2c and f77"), + 'lto':(False, "Enable Link Time Optimization"), + } COMPILER_UNIQUE_OPTION_MAP = { - 'i8': 'fdefault-integer-8', - 'r8': 'fdefault-real-8', - 'unroll': 'funroll-loops', - 'f2c': 'ff2c', - 'loop': ['ftree-switch-conversion', 'floop-interchange', - 'floop-strip-mine', 'floop-block'], - 'lto':'flto', - 'optarch':'march=native', - 'openmp':'fopenmp', - 'strict': ['mieee-fp', 'mno-recip'], - 'precise':['mno-recip'], - 'defaultprec':[], - 'loose': ['mrecip', 'mno-ieee-fp'], - 'veryloose': ['mrecip=all', 'mno-ieee-fp'], - } + 'i8': 'fdefault-integer-8', + 'r8': 'fdefault-real-8', + 'unroll': 'funroll-loops', + 'f2c': 'ff2c', + 'loop': ['ftree-switch-conversion', 'floop-interchange', 'floop-strip-mine', 'floop-block'], + 'lto': 'flto', + 'openmp': 'fopenmp', + 'strict': ['mieee-fp', 'mno-recip'], + 'precise':['mno-recip'], + 'defaultprec':[], + 'loose': ['mrecip', 'mno-ieee-fp'], + 'veryloose': ['mrecip=all', 'mno-ieee-fp'], + } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { - systemtools.INTEL : 'march=native', - systemtools.AMD : 'march=native' - } + systemtools.AMD : 'march=native', + systemtools.INTEL : 'march=native', + systemtools.POWER: 'mcpu=native', # no support for march=native on POWER + } COMPILER_CC = 'gcc' COMPILER_CXX = 'g++' diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 2d2fbc696c..921c8e926e 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -36,11 +36,7 @@ import sys from socket import gethostname from vsc.utils import fancylogger -try: - # this import fails with Python 2.4 because it requires the ctypes module (only in Python 2.5+) - from vsc.utils.affinity import sched_getaffinity -except ImportError: - pass +from vsc.utils.affinity import sched_getaffinity from easybuild.tools.filetools import read_file, which from easybuild.tools.run import run_cmd @@ -51,13 +47,26 @@ # constants AMD = 'AMD' ARM = 'ARM' +IBM = 'IBM' INTEL = 'Intel' +POWER = 'POWER' LINUX = 'Linux' DARWIN = 'Darwin' UNKNOWN = 'UNKNOWN' +MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' +PROC_CPUINFO_FP = '/proc/cpuinfo' + +CPU_FAMILIES = [ARM, AMD, INTEL, POWER] +VENDORS = { + 'ARM': ARM, + 'AuthenticAMD': AMD, + 'GenuineIntel': INTEL, + 'IBM': IBM, +} + class SystemToolsException(Exception): """raised when systemtools fails""" @@ -67,64 +76,25 @@ def get_avail_core_count(): """ Returns the number of available CPUs, according to cgroups and taskssets limits """ - # tiny inner function to help figure out number of available cores in a cpuset - def count_bits(n): - """Count the number of set bits for a given integer.""" - bit_cnt = 0 - while n > 0: - n &= n - 1 - bit_cnt += 1 - return bit_cnt - + core_cnt = None os_type = get_os_type() - if os_type == LINUX: - try: - # the preferred approach is via sched_getaffinity (yields a long, so cast it down to int) - num_cores = int(sum(sched_getaffinity().cpus)) - return num_cores - except NameError: - pass - # in case sched_getaffinity isn't available, fall back to relying on /proc/cpuinfo - - # determine total number of cores via /proc/cpuinfo - try: - txt = read_file('/proc/cpuinfo', log_error=False) - # sometimes this is uppercase - max_num_cores = txt.lower().count('processor\t:') - except IOError, err: - raise SystemToolsException("An error occured while determining total core count: %s" % err) - - # determine cpuset we're in (if any) - mypid = os.getpid() - try: - f = open("/proc/%s/status" % mypid, 'r') - txt = f.read() - f.close() - cpuset = re.search("^Cpus_allowed:\s*([0-9,a-f]+)", txt, re.M | re.I) - except IOError: - cpuset = None - - if cpuset is not None: - # use cpuset mask to determine actual number of available cores - mask_as_int = long(cpuset.group(1).replace(',', ''), 16) - num_cores_in_cpuset = count_bits((2**max_num_cores - 1) & mask_as_int) - _log.info("In cpuset with %s CPUs" % num_cores_in_cpuset) - return num_cores_in_cpuset - else: - _log.debug("No list of allowed CPUs found, not in a cpuset.") - return max_num_cores + if os_type == LINUX: + # simple use available sched_getaffinity() function (yields a long, so cast it down to int) + core_cnt = int(sum(sched_getaffinity().cpus)) else: - # BSD + # BSD-type systems + out, _ = run_cmd('sysctl -n hw.ncpu') try: - out, _ = run_cmd('sysctl -n hw.ncpu') - num_cores = int(out) - if num_cores > 0: - return num_cores + if int(out) > 0: + core_cnt = int(out) except ValueError: pass - raise SystemToolsException('Can not determine number of cores on this system') + if core_cnt is None: + raise SystemToolsException('Can not determine number of cores on this system') + else: + return core_cnt def get_core_count(): @@ -133,76 +103,100 @@ def get_core_count(): def get_cpu_vendor(): - """Try to detect the cpu identifier + """ + Try to detect the CPU vendor - will return INTEL, ARM or AMD constant + @return: a value from the VENDORS dict """ - regexp = re.compile(r"^vendor_id\s+:\s*(?P\S+)\s*$", re.M) - VENDORS = { - 'GenuineIntel': INTEL, - 'AuthenticAMD': AMD, - } + vendor = None os_type = get_os_type() - if os_type == LINUX: - try: - txt = read_file('/proc/cpuinfo', log_error=False) - arch = UNKNOWN - # vendor_id might not be in the /proc/cpuinfo, so this might fail - res = regexp.search(txt) - if res: - arch = res.groupdict().get('vendorid', UNKNOWN) - if arch in VENDORS: - return VENDORS[arch] + if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): + txt = read_file(PROC_CPUINFO_FP) + arch = UNKNOWN - # some embeded linux on arm behaves differently (e.g. raspbian) - regexp = re.compile(r"^Processor\s+:\s*(?PARM\S+)\s*", re.M) - res = regexp.search(txt) - if res: - arch = res.groupdict().get('vendorid', UNKNOWN) - if ARM in arch: - return ARM - except IOError, err: - raise SystemToolsException("An error occured while determining CPU vendor since: %s" % err) + vendor_regex = re.compile(r"(vendor_id.*?)?\s*:\s*(?P(?(1)\S+|(?:IBM|ARM)))") + res = vendor_regex.search(txt) + if res: + arch = res.group('vendor') + if arch in VENDORS: + vendor = VENDORS[arch] + tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP) + _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup) elif os_type == DARWIN: - out, exitcode = run_cmd("sysctl -n machdep.cpu.vendor") + cmd = "sysctl -n machdep.cpu.vendor" + out, ec = run_cmd(cmd) out = out.strip() - if not exitcode and out and out in VENDORS: - return VENDORS[out] + if ec == 0 and out in VENDORS: + vendor = VENDORS[out] + _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) + + if vendor is None: + vendor = UNKNOWN + _log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor)) + + return vendor + + +def get_cpu_family(): + """ + Determine CPU family. + @return: a value from the CPU_FAMILIES list + """ + family = None + vendor = get_cpu_vendor() + if vendor in CPU_FAMILIES: + family = vendor + _log.debug("Using vendor as CPU family: %s" % family) else: - # BSD - out, exitcode = run_cmd("sysctl -n hw.model") - out = out.strip() - if not exitcode and out: - return out.split(' ')[0] + # POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo + if os.path.exists(PROC_CPUINFO_FP): + cpuinfo_txt = read_file(PROC_CPUINFO_FP) + power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M) + if power_regex.search(cpuinfo_txt): + family = POWER + tup = (power_regex.pattern, PROC_CPUINFO_FP, family) + _log.debug("Determined CPU family using regex '%s' in %s: %s" % tup) - return UNKNOWN + if family is None: + family = UNKNOWN + _log.warning("Failed to determine CPU family, returning %s" % family) + + return family def get_cpu_model(): """ - returns cpu model - f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz + Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz """ + model = None os_type = get_os_type() - if os_type == LINUX: - regexp = re.compile(r"^model name\s+:\s*(?P.+)\s*$", re.M) - try: - txt = read_file('/proc/cpuinfo', log_error=False) - if txt is not None: - return regexp.search(txt).groupdict()['modelname'].strip() - except IOError, err: - raise SystemToolsException("An error occured when determining CPU model: %s" % err) + + if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): + # we need 'model name' on Linux/x86, but 'model' is there first with different info + # 'model name' is not there for Linux/POWER, but 'model' has the right info + model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P.*[A-Za-z].+)\s*$", re.M) + txt = read_file(PROC_CPUINFO_FP) + res = model_regex.search(txt) + if res is not None: + model = res.group('model').strip() + tup = (model_regex.pattern, PROC_CPUINFO_FP, model) + _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s" % tup) elif os_type == DARWIN: - out, exitcode = run_cmd("sysctl -n machdep.cpu.brand_string") - out = out.strip() - if not exitcode: - return out + cmd = "sysctl -n machdep.cpu.brand_string" + out, ec = run_cmd(cmd) + if ec == 0: + model = out.strip() + _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) + + if model is None: + model = UNKNOWN + _log.warning("Failed to determine CPU model, returning %s" % model) - return UNKNOWN + return model def get_cpu_speed(): @@ -210,48 +204,43 @@ def get_cpu_speed(): Returns the (maximum) cpu speed in MHz, as a float value. In case of throttling, the highest cpu speed is returns. """ + cpu_freq = None os_type = get_os_type() - if os_type == LINUX: - try: - # Linux with cpu scaling - max_freq_fp = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' - try: - f = open(max_freq_fp, 'r') - cpu_freq = float(f.read())/1000 - f.close() - return cpu_freq - except IOError, err: - _log.warning("Failed to read %s to determine max. CPU clock frequency with CPU scaling: %s" % (max_freq_fp, err)) - - # Linux without cpu scaling - cpuinfo_fp = '/proc/cpuinfo' - try: - cpu_freq = None - f = open(cpuinfo_fp, 'r') - for line in f: - cpu_freq = re.match("^cpu MHz\s*:\s*([0-9.]+)", line) - if cpu_freq is not None: - break - f.close() - if cpu_freq is None: - raise SystemToolsException("Failed to determine CPU frequency from %s" % cpuinfo_fp) - else: - return float(cpu_freq.group(1)) - except IOError, err: - _log.warning("Failed to read %s to determine CPU clock frequency: %s" % (cpuinfo_fp, err)) - except (IOError, OSError), err: - raise SystemToolsException("Determining CPU speed failed, exception occured: %s" % err) + if os_type == LINUX: + # Linux with cpu scaling + if os.path.exists(MAX_FREQ_FP): + _log.debug("Trying to determine CPU frequency on Linux via %s" % MAX_FREQ_FP) + txt = read_file(MAX_FREQ_FP) + cpu_freq = float(txt)/1000 + + # Linux without cpu scaling + elif os.path.exists(PROC_CPUINFO_FP): + _log.debug("Trying to determine CPU frequency on Linux via %s" % PROC_CPUINFO_FP) + cpuinfo_txt = read_file(PROC_CPUINFO_FP) + # 'cpu MHz' on Linux/x86 (& more), 'clock' on Linux/POWER + cpu_freq_regex = re.compile(r"^(?:cpu MHz|clock)\s*:\s*(?P\d+(?:\.\d+)?)", re.M) + res = cpu_freq_regex.search(cpuinfo_txt) + if res: + cpu_freq = float(res.group('cpu_freq')) + _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex.pattern, cpu_freq)) + else: + raise SystemToolsException("Failed to determine CPU frequency from %s" % PROC_CPUINFO_FP) + else: + _log.debug("%s not found to determine max. CPU clock frequency without CPU scaling: %s" % PROC_CPUINFO_FP) elif os_type == DARWIN: - # OS X - out, ec = run_cmd("sysctl -n hw.cpufrequency_max") - # returns clock frequency in cycles/sec, but we want MHz - mhz = float(out.strip())/(1000**2) + cmd = "sysctl -n hw.cpufrequency_max" + _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) + out, ec = run_cmd(cmd) if ec == 0: - return mhz + # returns clock frequency in cycles/sec, but we want MHz + cpu_freq = float(out.strip())/(1000**2) + + else: + raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) - raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) + return cpu_freq def get_kernel_name(): @@ -425,7 +414,7 @@ def get_glibc_version(): return glibc_version else: tup = (glibc_ver_str, glibc_ver_regex.pattern) - _log.error("Failed to determine version from '%s' using pattern '%s'." % tup) + _log.error("Failed to determine glibc version from '%s' using pattern '%s'." % tup) else: # no glibc on OS X standard _log.debug("No glibc on a non-Linux system, so can't determine version.") diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 9b72f621af..fda05be513 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -260,7 +260,7 @@ def _set_compiler_flags(self): def _get_optimal_architecture(self): """ Get options for the current architecture """ if self.arch is None: - self.arch = systemtools.get_cpu_vendor() + self.arch = systemtools.get_cpu_family() optarch = None if build_option('optarch') is not None: diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 8447913dc7..c33319e96a 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -27,53 +27,325 @@ @author: Kenneth hoste (Ghent University) """ -import os -from test.framework.utilities import EnhancedTestCase, init_config +import re +from os.path import exists as orig_os_path_exists +from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main -from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN -from easybuild.tools.systemtools import get_avail_core_count, get_core_count +import easybuild.tools.systemtools as st +from easybuild.tools.filetools import read_file +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import CPU_FAMILIES, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN, VENDORS +from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext from easybuild.tools.systemtools import get_system_info +MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' +PROC_CPUINFO_FP = '/proc/cpuinfo' + +PROC_CPUINFO_TXT = None +PROC_CPUINFO_TXT_ARM = """processor : 0 +model name : ARMv7 Processor rev 5 (v7l) +BogoMIPS : 57.60 +Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm +CPU implementer : 0x41 +CPU architecture: 7 +CPU variant : 0x0 +CPU part : 0xc07 +CPU revision : 5 + +processor : 1 +model name : ARMv7 Processor rev 5 (v7l) +BogoMIPS : 57.60 +Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm +CPU implementer : 0x41 +CPU architecture: 7 +CPU variant : 0x0 +CPU part : 0xc07 +CPU revision : 5 +""" +PROC_CPUINFO_TXT_POWER = """processor : 0 +cpu : POWER7 (architected), altivec supported +clock : 3550.000000MHz +revision : 2.3 (pvr 003f 0203) + +processor : 13 +cpu : POWER7 (architected), altivec supported +clock : 3550.000000MHz +revision : 2.3 (pvr 003f 0203) + +timebase : 512000000 +platform : pSeries +model : IBM,8205-E6C +machine : CHRP IBM,8205-E6C +""" +PROC_CPUINFO_TXT_X86 = """processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 45 +model name : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz +stepping : 7 +microcode : 1808 +cpu MHz : 2600.075 +cache size : 20480 KB +physical id : 0 +siblings : 8 +core id : 0 +cpu cores : 8 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid +bogomips : 5200.15 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 45 +model name : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz +stepping : 7 +microcode : 1808 +cpu MHz : 2600.075 +cache size : 20480 KB +physical id : 1 +siblings : 8 +core id : 0 +cpu cores : 8 +apicid : 32 +initial apicid : 32 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid +bogomips : 5200.04 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: +""" + + +def mocked_read_file(fp): + """Mocked version of read_file, with specified contents for known filenames.""" + known_fps = { + MAX_FREQ_FP: '2850000', + PROC_CPUINFO_FP: PROC_CPUINFO_TXT, + } + if fp in known_fps: + return known_fps[fp] + else: + return read_file(fp) + +def mocked_os_path_exists(mocked_fp, fp): + """Mocked version of os.path.exists, returns True for a particular specified filepath.""" + return fp == mocked_fp + +def mocked_run_cmd(cmd, **kwargs): + """Mocked version of run_cmd, with specified output for known commands.""" + known_cmds = { + "ldd --version" : "ldd (GNU libc) 2.12", + "sysctl -n hw.cpufrequency_max": "2400000000", + "sysctl -n hw.ncpu": '10', + "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz", + "sysctl -n machdep.cpu.vendor": 'GenuineIntel', + "ulimit -u": '40', + } + if cmd in known_cmds: + if 'simple' in kwargs and kwargs['simple']: + return True + else: + return (known_cmds[cmd], 0) + else: + return run_cmd(cmd, **kwargs) + + class SystemToolsTest(EnhancedTestCase): """ very basis FileRepository test, we don't want git / svn dependency """ - def test_avail_core_count(self): + def setUp(self): + """Set up systemtools test.""" + super(SystemToolsTest, self).setUp() + self.orig_get_os_type = st.get_os_type + self.orig_os_path_exists = st.os.path.exists + self.orig_read_file = st.read_file + self.orig_run_cmd = st.run_cmd + + def tearDown(self): + """Cleanup after systemtools test.""" + st.os.path.exists = self.orig_os_path_exists + st.read_file = self.orig_read_file + st.get_os_type = self.orig_get_os_type + st.run_cmd = self.orig_run_cmd + super(SystemToolsTest, self).tearDown() + + def test_avail_core_count_native(self): """Test getting core count.""" core_count = get_avail_core_count() self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) - def test_cpu_model(self): + def test_avail_core_count_linux(self): + """Test getting core count (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + orig_sched_getaffinity = st.sched_getaffinity + class MockedSchedGetaffinity(object): + cpus = [1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L] + st.sched_getaffinity = lambda: MockedSchedGetaffinity() + self.assertEqual(get_avail_core_count(), 6) + st.sched_getaffinity = orig_sched_getaffinity + + def test_avail_core_count_darwin(self): + """Test getting core count (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_avail_core_count(), 10) + + def test_cpu_model_native(self): """Test getting CPU model.""" cpu_model = get_cpu_model() self.assertTrue(isinstance(cpu_model, basestring)) - def test_cpu_speed(self): + def test_cpu_model_linux(self): + """Test getting CPU model (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + global PROC_CPUINFO_TXT + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_model(), "Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz") + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_model(), "IBM,8205-E6C") + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_model(), "ARMv7 Processor rev 5 (v7l)") + + def test_cpu_model_darwin(self): + """Test getting CPU model (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz") + + def test_cpu_speed_native(self): """Test getting CPU speed.""" cpu_speed = get_cpu_speed() self.assertTrue(isinstance(cpu_speed, float)) self.assertTrue(cpu_speed > 0.0) + def test_cpu_speed_linux(self): + """Test getting CPU speed (mocked for Linux).""" + # test for particular type of system by mocking used functions + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + + # tweak global constant used by mocked_read_file + global PROC_CPUINFO_TXT + + # /proc/cpuinfo on Linux x86 (no cpufreq) + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_speed(), 2600.075) + + # /proc/cpuinfo on Linux POWER + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_speed(), 3550.0) + + # Linux (x86) with cpufreq + st.os.path.exists = lambda fp: mocked_os_path_exists(MAX_FREQ_FP, fp) + self.assertEqual(get_cpu_speed(), 2850.0) + + def test_cpu_speed_darwin(self): + """Test getting CPU speed (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_speed(), 2400.0) + def test_cpu_vendor(self): """Test getting CPU vendor.""" cpu_vendor = get_cpu_vendor() - self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN]) + self.assertTrue(cpu_vendor in VENDORS.values() + [UNKNOWN]) + + def test_cpu_vendor_linux(self): + """Test getting CPU vendor (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + + global PROC_CPUINFO_TXT + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_vendor(), INTEL) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_vendor(), IBM) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_vendor(), ARM) + + def test_cpu_vendor_darwin(self): + """Test getting CPU vendor (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_vendor(), INTEL) + + def test_cpu_family_native(self): + """Test get_cpu_family function.""" + cpu_family = get_cpu_family() + self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN) + + def test_cpu_family_linux(self): + """Test get_cpu_family function (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + global PROC_CPUINFO_TXT + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_family(), INTEL) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_family(), ARM) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_family(), POWER) + + def test_cpu_family_darwin(self): + """Test get_cpu_family function (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_family(), INTEL) def test_os_type(self): """Test getting OS type.""" os_type = get_os_type() self.assertTrue(os_type in [DARWIN, LINUX]) - def test_shared_lib_ext(self): + def test_shared_lib_ext_native(self): """Test getting extension for shared libraries.""" ext = get_shared_lib_ext() self.assertTrue(ext in ['dylib', 'so']) - def test_platform_name(self): + def test_shared_lib_ext_native(self): + """Test getting extension for shared libraries (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + self.assertEqual(get_shared_lib_ext(), 'so') + + def test_shared_lib_ext_native(self): + """Test getting extension for shared libraries (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertEqual(get_shared_lib_ext(), 'dylib') + + def test_platform_name_native(self): """Test getting platform name.""" platform_name_nover = get_platform_name() self.assertTrue(isinstance(platform_name_nover, basestring)) @@ -86,6 +358,18 @@ def test_platform_name(self): self.assertTrue(platform_name_ver.startswith(platform_name_ver)) self.assertTrue(len_ver >= len_nover) + def test_platform_name_linux(self): + """Test getting platform name (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + self.assertTrue(re.match('.*-unknown-linux$', get_platform_name())) + self.assertTrue(re.match('.*-unknown-linux-gnu$', get_platform_name(withversion=True))) + + def test_platform_name_darwin(self): + """Test getting platform name (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertTrue(re.match('.*-apple-darwin$', get_platform_name())) + self.assertTrue(re.match('.*-apple-darwin.*$', get_platform_name(withversion=True))) + def test_os_name(self): """Test getting OS name.""" os_name = get_os_name() @@ -96,16 +380,53 @@ def test_os_version(self): os_version = get_os_version() self.assertTrue(isinstance(os_version, basestring) or os_version == UNKNOWN) - def test_glibc_version(self): + def test_glibc_version_native(self): """Test getting glibc version.""" glibc_version = get_glibc_version() self.assertTrue(isinstance(glibc_version, basestring) or glibc_version == UNKNOWN) + def test_glibc_version_linux(self): + """Test getting glibc version (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.run_cmd = mocked_run_cmd + self.assertEqual(get_glibc_version(), '2.12') + + def test_glibc_version_darwin(self): + """Test getting glibc version (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertEqual(get_glibc_version(), UNKNOWN) + def test_system_info(self): """Test getting system info.""" system_info = get_system_info() self.assertTrue(isinstance(system_info, dict)) + def test_det_parallelism_native(self): + """Test det_parallelism function (native calls).""" + self.assertTrue(det_parallelism(None, None) > 0) + # specified parallellism + self.assertEqual(det_parallelism(5, None), 5) + # max parallellism caps + self.assertEqual(det_parallelism(None, 1), 1) + self.assertEqual(det_parallelism(16, 1), 1) + self.assertEqual(det_parallelism(5, 2), 2) + self.assertEqual(det_parallelism(5, 10), 5) + + def test_det_parallelism_mocked(self): + """Test det_parallelism function (with mocked ulimit/get_avail_core_count).""" + orig_get_avail_core_count = st.get_avail_core_count + + # mock number of available cores to 8 + st.get_avail_core_count = lambda: 8 + self.assertTrue(det_parallelism(None, None), 8) + # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) + st.run_cmd = mocked_run_cmd + self.assertTrue(det_parallelism(None, None), 4) + self.assertTrue(det_parallelism(6, None), 4) + self.assertTrue(det_parallelism(2, None), 2) + + st.get_avail_core_count = orig_get_avail_core_count + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(SystemToolsTest) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index cc30bc341a..e6e80ff5e4 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -42,6 +42,10 @@ from easybuild.tools.toolchain.utilities import search_toolchain from test.framework.utilities import find_full_path +from easybuild.tools import systemtools as st +import easybuild.tools.toolchain.compiler +easybuild.tools.toolchain.compiler.systemtools.get_compiler_family = lambda: st.POWER + class ToolchainTest(EnhancedTestCase): """ Baseclass for toolchain testcases """ @@ -284,7 +288,10 @@ def test_misc_flags_unique(self): tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() - flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] + if opt == 'optarch': + flag = '-%s' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch] + else: + flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: @@ -307,7 +314,8 @@ def test_override_optarch(self): if optarch_var is not None: flag = '-%s' % optarch_var else: - flag = '-march=native' + # default optarch flag + flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch] for var in flag_vars: flags = tc.get_variable(var) @@ -413,7 +421,7 @@ def test_goolfc(self): tc.prepare() nvcc_flags = r' '.join([ - r'-Xcompiler="-O2 -march=native"', + r'-Xcompiler="-O2 -%s"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch], # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]),