From f3db58ed33b600bbdb60997e3d8c187b930764cd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 12 Sep 2014 17:34:41 +0200
Subject: [PATCH 01/21] fix get_cpu_model in case /proc/cpuinfo doesn't list a
 model name

---
 easybuild/tools/systemtools.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 7cd2babc04..f908673ae0 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -190,13 +190,16 @@ def get_cpu_model():
     returns cpu model
     f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
     """
+    model = UNKNOWN
     os_type = get_os_type()
     if os_type == LINUX:
         regexp = re.compile(r"^model name\s+:\s*(?P<modelname>.+)\s*$", re.M)
         try:
             txt = read_file('/proc/cpuinfo', log_error=False)
             if txt is not None:
-                return regexp.search(txt).groupdict()['modelname'].strip()
+                res = regexp.search(txt)
+                if res is not None:
+                    model = res.group('modelname').strip()
         except IOError, err:
             raise SystemToolsException("An error occured when determining CPU model: %s" % err)
 
@@ -204,9 +207,9 @@ def get_cpu_model():
         out, exitcode = run_cmd("sysctl -n machdep.cpu.brand_string")
         out = out.strip()
         if not exitcode:
-            return out
+            model = out
 
-    return UNKNOWN
+    return model
 
 
 def get_cpu_speed():

From b2ab456a3adf0a3ee2b29e6c53ce9e16a8e1ee0f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 12 Sep 2014 19:09:33 +0200
Subject: [PATCH 02/21] enhance get_cpu_speed to also support determining CPU
 frequency on POWER, and clean things up

---
 easybuild/tools/systemtools.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index f908673ae0..3f9c24e807 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -222,30 +222,40 @@ def get_cpu_speed():
         try:
             # Linux with cpu scaling
             max_freq_fp = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
+            _log.debug("Trying to determine CPU frequency via %s" % max_freq_fp)
             try:
                 f = open(max_freq_fp, 'r')
                 cpu_freq = float(f.read())/1000
                 f.close()
                 return cpu_freq
             except IOError, err:
-                _log.warning("Failed to read %s to determine max. CPU clock frequency with CPU scaling: %s" % (max_freq_fp, err))
+                _log.debug("Failed to read %s to determine max. CPU clock frequency with CPU scaling: %s" % (max_freq_fp, err))
 
             # Linux without cpu scaling
             cpuinfo_fp = '/proc/cpuinfo'
+            _log.debug("Trying to determine CPU frequency via %s" % cpuinfo_fp)
             try:
                 cpu_freq = None
-                f = open(cpuinfo_fp, 'r')
-                for line in f:
-                    cpu_freq = re.match("^cpu MHz\s*:\s*([0-9.]+)", line)
-                    if cpu_freq is not None:
+                cpuinfo_txt = open(cpuinfo_fp, 'r').read()
+                cpu_freq_patterns = [
+                    r"^cpu MHz\s*:\s*(?P<cpu_freq>[0-9.]+)",  # Linux x86 & more
+                    r"^clock\s*:\s*(?P<cpu_freq>[0-9.]+)",  # Linux on POWER
+                ]
+                for cpu_freq_pattern in cpu_freq_patterns:
+                    cpu_freq_re = re.compile(cpu_freq_pattern, re.M)
+                    res = cpu_freq_re.search(cpuinfo_txt)
+                    if res:
+                        cpu_freq = res.group('cpu_freq')
+                        _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_pattern, cpu_freq))
                         break
-                f.close()
+                    else:
+                        _log.debug("Failed to determine CPU frequency using regex '%s'" % cpu_freq_re.pattern)
                 if cpu_freq is None:
                     raise SystemToolsException("Failed to determine CPU frequency from %s" % cpuinfo_fp)
                 else:
-                    return float(cpu_freq.group(1))
+                    return float(cpu_freq)
             except IOError, err:
-                _log.warning("Failed to read %s to determine CPU clock frequency: %s" % (cpuinfo_fp, err))
+                _log.debug("Failed to read %s to determine CPU clock frequency: %s" % (cpuinfo_fp, err))
 
         except (IOError, OSError), err:
             raise SystemToolsException("Determining CPU speed failed, exception occured: %s" % err)

From 00bfdb55427ffdf06b577d51d794fd43bb00fca0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 09:06:41 +0100
Subject: [PATCH 03/21] clean up implementation of get_cpu_speed

---
 easybuild/tools/systemtools.py | 90 ++++++++++++++++------------------
 1 file changed, 41 insertions(+), 49 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 77dc04b996..ea72b64156 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -58,6 +58,9 @@
 
 UNKNOWN = 'UNKNOWN'
 
+MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
+PROC_CPUINFO_FP = '/proc/cpuinfo'
+
 
 class SystemToolsException(Exception):
     """raised when systemtools fails"""
@@ -89,7 +92,7 @@ def count_bits(n):
 
         # determine total number of cores via /proc/cpuinfo
         try:
-            txt = read_file('/proc/cpuinfo', log_error=False)
+            txt = read_file(PROC_CPUINFO_FP, log_error=False)
             # sometimes this is uppercase
             max_num_cores = txt.lower().count('processor\t:')
         except IOError, err:
@@ -146,7 +149,7 @@ def get_cpu_vendor():
 
     if os_type == LINUX:
         try:
-            txt = read_file('/proc/cpuinfo', log_error=False)
+            txt = read_file(PROC_CPUINFO_FP, log_error=False)
             arch = UNKNOWN
             # vendor_id might not be in the /proc/cpuinfo, so this might fail
             res = regexp.search(txt)
@@ -191,7 +194,7 @@ def get_cpu_model():
     if os_type == LINUX:
         regexp = re.compile(r"^model name\s+:\s*(?P<modelname>.+)\s*$", re.M)
         try:
-            txt = read_file('/proc/cpuinfo', log_error=False)
+            txt = read_file(PROC_CPUINFO_FP, log_error=False)
             if txt is not None:
                 res = regexp.search(txt)
                 if res is not None:
@@ -213,58 +216,47 @@ def get_cpu_speed():
     Returns the (maximum) cpu speed in MHz, as a float value.
     In case of throttling, the highest cpu speed is returns.
     """
+    cpu_freq = None
     os_type = get_os_type()
+
     if os_type == LINUX:
-        try:
-            # Linux with cpu scaling
-            max_freq_fp = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
-            _log.debug("Trying to determine CPU frequency via %s" % max_freq_fp)
-            try:
-                f = open(max_freq_fp, 'r')
-                cpu_freq = float(f.read())/1000
-                f.close()
-                return cpu_freq
-            except IOError, err:
-                _log.debug("Failed to read %s to determine max. CPU clock frequency with CPU scaling: %s" % (max_freq_fp, err))
-
-            # Linux without cpu scaling
-            cpuinfo_fp = '/proc/cpuinfo'
-            _log.debug("Trying to determine CPU frequency via %s" % cpuinfo_fp)
-            try:
-                cpu_freq = None
-                cpuinfo_txt = open(cpuinfo_fp, 'r').read()
-                cpu_freq_patterns = [
-                    r"^cpu MHz\s*:\s*(?P<cpu_freq>[0-9.]+)",  # Linux x86 & more
-                    r"^clock\s*:\s*(?P<cpu_freq>[0-9.]+)",  # Linux on POWER
-                ]
-                for cpu_freq_pattern in cpu_freq_patterns:
-                    cpu_freq_re = re.compile(cpu_freq_pattern, re.M)
-                    res = cpu_freq_re.search(cpuinfo_txt)
-                    if res:
-                        cpu_freq = res.group('cpu_freq')
-                        _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_pattern, cpu_freq))
-                        break
-                    else:
-                        _log.debug("Failed to determine CPU frequency using regex '%s'" % cpu_freq_re.pattern)
-                if cpu_freq is None:
-                    raise SystemToolsException("Failed to determine CPU frequency from %s" % cpuinfo_fp)
-                else:
-                    return float(cpu_freq)
-            except IOError, err:
-                _log.debug("Failed to read %s to determine CPU clock frequency: %s" % (cpuinfo_fp, err))
-
-        except (IOError, OSError), err:
-            raise SystemToolsException("Determining CPU speed failed, exception occured: %s" % err)
+        # Linux with cpu scaling
+        if os.path.exists(MAX_FREQ_FP):
+            _log.debug("Trying to determine CPU frequency on Linux via %s" % MAX_FREQ_FP)
+            txt = read_file(MAX_FREQ_FP)
+            cpu_freq = float(txt)/1000
+
+        # Linux without cpu scaling
+        elif os.path.exists(PROC_CPUINFO_FP):
+            _log.debug("Trying to determine CPU frequency on Linux via %s" % PROC_CPUINFO_FP)
+            cpuinfo_txt = read_file(PROC_CPUINFO_FP)
+            cpu_freq_regex = r"(%s)" % '|'.join([
+                r"^cpu MHz\s*:\s*(?P<cpu_freq_x86>[0-9.]+)",  # Linux x86 & more
+                r"^clock\s*:\s*(?P<cpu_freq_POWER>[0-9.]+)",  # Linux on POWER
+            ])
+            res = re.search(cpu_freq_regex, cpuinfo_txt, re.M)
+            if res:
+                cpu_freq = res.group('cpu_freq_x86') or res.group('cpu_freq_POWER')
+            if cpu_freq is not None:
+                cpu_freq = float(cpu_freq)
+                _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex, cpu_freq))
+            else:
+                raise SystemToolsException("Failed to determine CPU frequency from %s" % PROC_CPUINFO_FP)
+        else:
+            _log.debug("%s not found to determine max. CPU clock frequency without CPU scaling: %s" % PROC_CPUINFO_FP)
 
     elif os_type == DARWIN:
-        # OS X
-        out, ec = run_cmd("sysctl -n hw.cpufrequency_max")
-        # returns clock frequency in cycles/sec, but we want MHz
-        mhz = float(out.strip())/(1000**2)
+        cmd = "sysctl -n hw.cpufrequency_max"
+        _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
+        out, ec = run_cmd(cmd)
         if ec == 0:
-            return mhz
+            # returns clock frequency in cycles/sec, but we want MHz
+            cpu_freq = float(out.strip())/(1000**2)
+
+    else:
+        raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type)
 
-    raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type)
+    return cpu_freq
 
 
 def get_kernel_name():

From 6f900a40f576e0e07c1df7ef2be2d5fa5a9cbfe1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 09:08:38 +0100
Subject: [PATCH 04/21] thoroughly test possible code paths in get_cpu_speed by
 mocking used functions

---
 test/framework/systemtools.py | 155 +++++++++++++++++++++++++++++++++-
 1 file changed, 153 insertions(+), 2 deletions(-)

diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index 8447913dc7..95c7bbf81b 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -28,19 +28,144 @@
 @author: Kenneth hoste (Ghent University)
 """
 import os
-from test.framework.utilities import EnhancedTestCase, init_config
+from os.path import exists as orig_os_path_exists
+from test.framework.utilities import EnhancedTestCase
 from unittest import TestLoader, main
 
+import easybuild.tools.systemtools as st
+from easybuild.tools.filetools import read_file
+from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN
-from easybuild.tools.systemtools import get_avail_core_count, get_core_count
+from easybuild.tools.systemtools import get_avail_core_count
 from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version
 from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext
 from easybuild.tools.systemtools import get_system_info
 
 
+MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
+PROC_CPUINFO_FP = '/proc/cpuinfo'
+
+PROC_CPUINFO_TXT = None
+PROC_CPUINFO_TXT_X86 = """processor	: 0
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 45
+model name	: Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz
+stepping	: 7
+microcode	: 1808
+cpu MHz		: 2600.075
+cache size	: 20480 KB
+physical id	: 0
+siblings	: 8
+core id		: 0
+cpu cores	: 8
+apicid		: 0
+initial apicid	: 0
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 13
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5200.15
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 46 bits physical, 48 bits virtual
+power management:
+
+processor	: 1
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 45
+model name	: Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz
+stepping	: 7
+microcode	: 1808
+cpu MHz		: 2600.075
+cache size	: 20480 KB
+physical id	: 1
+siblings	: 8
+core id		: 0
+cpu cores	: 8
+apicid		: 32
+initial apicid	: 32
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 13
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5200.04
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 46 bits physical, 48 bits virtual
+power management:
+"""
+PROC_CPUINFO_TXT_POWER = """
+processor	: 0
+cpu		: POWER7 (architected), altivec supported
+clock		: 3550.000000MHz
+revision	: 2.3 (pvr 003f 0203)
+ 
+processor	: 13
+cpu		: POWER7 (architected), altivec supported
+clock		: 3550.000000MHz
+revision	: 2.3 (pvr 003f 0203)
+ 
+timebase	: 512000000
+platform	: pSeries
+model		: IBM,8205-E6C
+machine		: CHRP IBM,8205-E6C
+"""
+
+
+def mocked_read_file(fp):
+    """Mocked version of read_file, with specified contents for known filenames."""
+    known_fps = {
+        MAX_FREQ_FP:  '2850000',
+        PROC_CPUINFO_FP: PROC_CPUINFO_TXT,
+    }
+    if fp in known_fps:
+        return known_fps[fp]
+    else:
+        return read_file(fp)
+
+def mocked_os_path_exists(mocked_fp, fp):
+    """Mocked version of os.path.exists, returns True for a particular specified filepath."""
+    if fp == mocked_fp:
+        return True
+    else:
+        orig_os_path_exists(fp)
+
+def mocked_run_cmd(cmd, **kwargs):
+    """Mocked version of run_cmd, with specified output for known commands."""
+    known_cmds = {
+        "sysctl -n hw.cpufrequency_max": "2400000000",
+    }
+    if cmd in known_cmds:
+        if 'simple' in kwargs and kwargs['simple']:
+            return True
+        else:
+            return (known_cmds[cmd], 0)
+    else:
+        return run_cmd(cmd, **kwargs)
+
+
 class SystemToolsTest(EnhancedTestCase):
     """ very basis FileRepository test, we don't want git / svn dependency """
 
+    def setUp(self):
+        """Set up systemtools test."""
+        super(SystemToolsTest, self).setUp()
+        self.orig_get_os_type = st.get_os_type
+        self.orig_os_path_exists = st.os.path.exists
+        self.orig_read_file = st.read_file
+        self.orig_run_cmd = st.run_cmd
+
+    def tearDown(self):
+        """Cleanup after systemtools test."""
+        st.os.path.exists = self.orig_os_path_exists
+        st.read_file = self.orig_read_file
+        st.get_os_type = self.orig_get_os_type
+        super(SystemToolsTest, self).tearDown()
+
     def test_avail_core_count(self):
         """Test getting core count."""
         core_count = get_avail_core_count()
@@ -58,6 +183,32 @@ def test_cpu_speed(self):
         self.assertTrue(isinstance(cpu_speed, float))
         self.assertTrue(cpu_speed > 0.0)
 
+        # test for particular type of system by mocking used functions
+        st.get_os_type = lambda: st.LINUX
+        st.read_file = mocked_read_file
+        st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
+
+        # tweak global constant used by mocked_read_file
+        global PROC_CPUINFO_TXT
+
+        # /proc/cpuinfo on Linux x86 (no cpufreq)
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
+        self.assertEqual(get_cpu_speed(), 2600.075)
+
+        # /proc/cpuinfo on Linux POWER
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
+        self.assertEqual(get_cpu_speed(), 3550.0)
+
+        # Linux (x86) with cpufreq
+        st.os.path.exists = lambda fp: mocked_os_path_exists(MAX_FREQ_FP, fp)
+        self.assertEqual(get_cpu_speed(), 2850.0)
+
+        # OS X
+        st.os.path.exists = self.orig_os_path_exists
+        st.get_os_type = lambda: st.DARWIN
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_cpu_speed(), 2400.0)
+
     def test_cpu_vendor(self):
         """Test getting CPU vendor."""
         cpu_vendor = get_cpu_vendor()

From 82f3b4a6eafab5cc5506f10a96517d8577056c68 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 09:27:06 +0100
Subject: [PATCH 05/21] clean up get_avail_core_count, enhance testing for it

---
 easybuild/tools/systemtools.py | 70 +++++++---------------------------
 test/framework/systemtools.py  | 13 +++++++
 2 files changed, 27 insertions(+), 56 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index ea72b64156..086bc62088 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -36,11 +36,7 @@
 import sys
 from socket import gethostname
 from vsc.utils import fancylogger
-try:
-    # this import fails with Python 2.4 because it requires the ctypes module (only in Python 2.5+)
-    from vsc.utils.affinity import sched_getaffinity
-except ImportError:
-    pass
+from vsc.utils.affinity import sched_getaffinity
 
 from easybuild.tools.filetools import read_file, which
 from easybuild.tools.run import run_cmd
@@ -52,6 +48,7 @@
 AMD = 'AMD'
 ARM = 'ARM'
 INTEL = 'Intel'
+PPC = 'PPC'
 
 LINUX = 'Linux'
 DARWIN = 'Darwin'
@@ -70,64 +67,25 @@ def get_avail_core_count():
     """
     Returns the number of available CPUs, according to cgroups and taskssets limits
     """
-    # tiny inner function to help figure out number of available cores in a cpuset
-    def count_bits(n):
-        """Count the number of set bits for a given integer."""
-        bit_cnt = 0
-        while n > 0:
-            n &= n - 1
-            bit_cnt += 1
-        return bit_cnt
-
+    core_cnt = None
     os_type = get_os_type()
-    if os_type == LINUX:
-        try:
-            # the preferred approach is via sched_getaffinity (yields a long, so cast it down to int)
-            num_cores = int(sum(sched_getaffinity().cpus))
-            return num_cores
-        except NameError:
-            pass
-
-        # in case sched_getaffinity isn't available, fall back to relying on /proc/cpuinfo
-
-        # determine total number of cores via /proc/cpuinfo
-        try:
-            txt = read_file(PROC_CPUINFO_FP, log_error=False)
-            # sometimes this is uppercase
-            max_num_cores = txt.lower().count('processor\t:')
-        except IOError, err:
-            raise SystemToolsException("An error occured while determining total core count: %s" % err)
 
-        # determine cpuset we're in (if any)
-        mypid = os.getpid()
-        try:
-            f = open("/proc/%s/status" % mypid, 'r')
-            txt = f.read()
-            f.close()
-            cpuset = re.search("^Cpus_allowed:\s*([0-9,a-f]+)", txt, re.M | re.I)
-        except IOError:
-            cpuset = None
-
-        if cpuset is not None:
-            # use cpuset mask to determine actual number of available cores
-            mask_as_int = long(cpuset.group(1).replace(',', ''), 16)
-            num_cores_in_cpuset = count_bits((2**max_num_cores - 1) & mask_as_int)
-            _log.info("In cpuset with %s CPUs" % num_cores_in_cpuset)
-            return num_cores_in_cpuset
-        else:
-            _log.debug("No list of allowed CPUs found, not in a cpuset.")
-            return max_num_cores
+    if os_type == LINUX:
+        # simple use available sched_getaffinity() function (yields a long, so cast it down to int)
+        core_cnt = int(sum(sched_getaffinity().cpus))
     else:
-        # BSD
+        # BSD-type systems
+        out, _ = run_cmd('sysctl -n hw.ncpu')
         try:
-            out, _ = run_cmd('sysctl -n hw.ncpu')
-            num_cores = int(out)
-            if num_cores > 0:
-                return num_cores
+            if int(out) > 0:
+                core_cnt = int(out)
         except ValueError:
             pass
 
-    raise SystemToolsException('Can not determine number of cores on this system')
+    if core_cnt is None:
+        raise SystemToolsException('Can not determine number of cores on this system')
+    else:
+        return core_cnt
 
 
 def get_core_count():
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index 95c7bbf81b..472247dc98 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -138,6 +138,7 @@ def mocked_run_cmd(cmd, **kwargs):
     """Mocked version of run_cmd, with specified output for known commands."""
     known_cmds = {
         "sysctl -n hw.cpufrequency_max": "2400000000",
+        "sysctl -n hw.ncpu": '10',
     }
     if cmd in known_cmds:
         if 'simple' in kwargs and kwargs['simple']:
@@ -172,6 +173,18 @@ def test_avail_core_count(self):
         self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count)))
         self.assertTrue(core_count > 0, "core_count %d > 0" % core_count)
 
+        st.get_os_type = lambda: st.LINUX
+        orig_sched_getaffinity = st.sched_getaffinity
+        class MockedSchedGetaffinity(object):
+            cpus = [1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L]
+        st.sched_getaffinity = lambda: MockedSchedGetaffinity()
+        self.assertEqual(get_avail_core_count(), 6)
+        st.sched_getaffinity = orig_sched_getaffinity
+
+        st.get_os_type = lambda: st.DARWIN
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_avail_core_count(), 10)
+
     def test_cpu_model(self):
         """Test getting CPU model."""
         cpu_model = get_cpu_model()

From 526c4fe2587d13dd87063068acc3fbe48a0f8c08 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 09:53:40 +0100
Subject: [PATCH 06/21] clean up get_cpu_vendor and enhance dedicated test for
 it

---
 easybuild/tools/systemtools.py | 76 +++++++++++++++++-----------------
 test/framework/systemtools.py  | 68 +++++++++++++++++++++++-------
 2 files changed, 91 insertions(+), 53 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 086bc62088..daff28ccae 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -58,6 +58,11 @@
 MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
 PROC_CPUINFO_FP = '/proc/cpuinfo'
 
+VENDORS = {
+    'GenuineIntel': INTEL,
+    'AuthenticAMD': AMD,
+}
+
 
 class SystemToolsException(Exception):
     """raised when systemtools fails"""
@@ -94,52 +99,49 @@ def get_core_count():
 
 
 def get_cpu_vendor():
-    """Try to detect the cpu identifier
+    """
+    Try to detect the CPU vendor
 
-    will return INTEL, ARM or AMD constant
+    @return: INTEL, ARM or AMD constant
     """
-    regexp = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M)
-    VENDORS = {
-        'GenuineIntel': INTEL,
-        'AuthenticAMD': AMD,
-    }
+    vendor = None
     os_type = get_os_type()
 
-    if os_type == LINUX:
-        try:
-            txt = read_file(PROC_CPUINFO_FP, log_error=False)
-            arch = UNKNOWN
-            # vendor_id might not be in the /proc/cpuinfo, so this might fail
-            res = regexp.search(txt)
-            if res:
-                arch = res.groupdict().get('vendorid', UNKNOWN)
-            if arch in VENDORS:
-                return VENDORS[arch]
-
-            # some embeded linux on arm behaves differently (e.g. raspbian)
-            regexp = re.compile(r"^Processor\s+:\s*(?P<vendorid>ARM\S+)\s*", re.M)
-            res = regexp.search(txt)
-            if res:
-                arch = res.groupdict().get('vendorid', UNKNOWN)
-            if ARM in arch:
-                return ARM
-        except IOError, err:
-            raise SystemToolsException("An error occured while determining CPU vendor since: %s" % err)
+    if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
+        txt = read_file(PROC_CPUINFO_FP)
+        arch = UNKNOWN
+
+        # vendor_id might not be in the /proc/cpuinfo, so this might fail
+        vendor_regex = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M)
+        res = vendor_regex.search(txt)
+        if res:
+            arch = res.group('vendorid')
+        if arch in VENDORS:
+            vendor = VENDORS[arch]
+            tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
+            _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup)
+
+        # embedded Linux on ARM behaves differently (e.g. Raspbian)
+        vendor_regex = re.compile(r".*:\s*(?P<vendorid>ARM\S+)\s*", re.M)
+        res = vendor_regex.search(txt)
+        if res:
+            vendor = ARM
+            tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
+            _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup)
 
     elif os_type == DARWIN:
-        out, exitcode = run_cmd("sysctl -n machdep.cpu.vendor")
+        cmd = "sysctl -n machdep.cpu.vendor"
+        out, ec = run_cmd(cmd)
         out = out.strip()
-        if not exitcode and out and out in VENDORS:
-            return VENDORS[out]
+        if ec == 0 and out and out in VENDORS:
+            vendor = VENDORS[out]
+            _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
 
-    else:
-        # BSD
-        out, exitcode = run_cmd("sysctl -n hw.model")
-        out = out.strip()
-        if not exitcode and out:
-            return out.split(' ')[0]
+    if vendor is None:
+        vendor = UNKNOWN
+        _log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor))
 
-    return UNKNOWN
+    return vendor
 
 
 def get_cpu_model():
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index 472247dc98..f49c38c90a 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -46,6 +46,41 @@
 PROC_CPUINFO_FP = '/proc/cpuinfo'
 
 PROC_CPUINFO_TXT = None
+PROC_CPUINFO_TXT_ARM = """processor : 0
+model name : ARMv7 Processor rev 5 (v7l)
+BogoMIPS : 57.60
+Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm
+CPU implementer : 0x41
+CPU architecture: 7
+CPU variant : 0x0
+CPU part : 0xc07
+CPU revision : 5
+ 
+processor : 1
+model name : ARMv7 Processor rev 5 (v7l)
+BogoMIPS : 57.60
+Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm
+CPU implementer : 0x41
+CPU architecture: 7
+CPU variant : 0x0
+CPU part : 0xc07
+CPU revision : 5 
+"""
+PROC_CPUINFO_TXT_POWER = """processor	: 0
+cpu		: POWER7 (architected), altivec supported
+clock		: 3550.000000MHz
+revision	: 2.3 (pvr 003f 0203)
+ 
+processor	: 13
+cpu		: POWER7 (architected), altivec supported
+clock		: 3550.000000MHz
+revision	: 2.3 (pvr 003f 0203)
+ 
+timebase	: 512000000
+platform	: pSeries
+model		: IBM,8205-E6C
+machine		: CHRP IBM,8205-E6C
+"""
 PROC_CPUINFO_TXT_X86 = """processor	: 0
 vendor_id	: GenuineIntel
 cpu family	: 6
@@ -98,22 +133,6 @@
 address sizes	: 46 bits physical, 48 bits virtual
 power management:
 """
-PROC_CPUINFO_TXT_POWER = """
-processor	: 0
-cpu		: POWER7 (architected), altivec supported
-clock		: 3550.000000MHz
-revision	: 2.3 (pvr 003f 0203)
- 
-processor	: 13
-cpu		: POWER7 (architected), altivec supported
-clock		: 3550.000000MHz
-revision	: 2.3 (pvr 003f 0203)
- 
-timebase	: 512000000
-platform	: pSeries
-model		: IBM,8205-E6C
-machine		: CHRP IBM,8205-E6C
-"""
 
 
 def mocked_read_file(fp):
@@ -139,6 +158,7 @@ def mocked_run_cmd(cmd, **kwargs):
     known_cmds = {
         "sysctl -n hw.cpufrequency_max": "2400000000",
         "sysctl -n hw.ncpu": '10',
+        "sysctl -n machdep.cpu.vendor": 'GenuineIntel',
     }
     if cmd in known_cmds:
         if 'simple' in kwargs and kwargs['simple']:
@@ -227,6 +247,22 @@ def test_cpu_vendor(self):
         cpu_vendor = get_cpu_vendor()
         self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN])
 
+        st.get_os_type = lambda: st.LINUX
+        st.read_file = mocked_read_file
+        st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
+
+        global PROC_CPUINFO_TXT
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
+        self.assertEqual(get_cpu_vendor(), INTEL)
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
+        self.assertEqual(get_cpu_vendor(), ARM)
+
+        st.os.path.exists = self.orig_os_path_exists
+        st.get_os_type = lambda: st.DARWIN
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_cpu_vendor(), INTEL)
+
     def test_os_type(self):
         """Test getting OS type."""
         os_type = get_os_type()

From 793af55e4e5d35438a7ad602c0a2f31450229339 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 10:34:28 +0100
Subject: [PATCH 07/21] clean up get_cpu_model and enhance dedicated test for
 it

---
 easybuild/tools/systemtools.py | 42 ++++++++++++++++++++--------------
 test/framework/systemtools.py  | 25 ++++++++++++++++----
 2 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index daff28ccae..52d8525ace 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -146,27 +146,35 @@ def get_cpu_vendor():
 
 def get_cpu_model():
     """
-    returns cpu model
-    f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
+    Determine CPU model
+    for example: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
     """
-    model = UNKNOWN
+    model = None
     os_type = get_os_type()
-    if os_type == LINUX:
-        regexp = re.compile(r"^model name\s+:\s*(?P<modelname>.+)\s*$", re.M)
-        try:
-            txt = read_file(PROC_CPUINFO_FP, log_error=False)
-            if txt is not None:
-                res = regexp.search(txt)
-                if res is not None:
-                    model = res.group('modelname').strip()
-        except IOError, err:
-            raise SystemToolsException("An error occured when determining CPU model: %s" % err)
+
+    if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
+        # consider 'model name' first, use 'model' as a fallback
+        # 'model name' is not there for Linux/POWER, but 'model' has the right info
+        for key in [r'model\s*name', 'model']:
+            model_regex = re.compile(r"^%s\s+:\s*(?P<model>.+)\s*$" % key, re.M)
+            txt = read_file(PROC_CPUINFO_FP)
+            res = model_regex.search(txt)
+            if res is not None:
+                model = res.group('model').strip()
+                tup = (model_regex.pattern, PROC_CPUINFO_FP, model)
+                _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s" % tup)
+                break
 
     elif os_type == DARWIN:
-        out, exitcode = run_cmd("sysctl -n machdep.cpu.brand_string")
-        out = out.strip()
-        if not exitcode:
-            model = out
+        cmd = "sysctl -n machdep.cpu.brand_string"
+        out, ec = run_cmd(cmd)
+        if ec == 0:
+            model = out.strip()
+            _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
+
+    if model is None:
+        model = UNKNOWN
+        _log.warning("Failed to determine CPU model, returning %s" % model)
 
     return model
 
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index f49c38c90a..091e19309b 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -148,16 +148,14 @@ def mocked_read_file(fp):
 
 def mocked_os_path_exists(mocked_fp, fp):
     """Mocked version of os.path.exists, returns True for a particular specified filepath."""
-    if fp == mocked_fp:
-        return True
-    else:
-        orig_os_path_exists(fp)
+    return fp == mocked_fp or orig_os_path_exists(fp)
 
 def mocked_run_cmd(cmd, **kwargs):
     """Mocked version of run_cmd, with specified output for known commands."""
     known_cmds = {
         "sysctl -n hw.cpufrequency_max": "2400000000",
         "sysctl -n hw.ncpu": '10',
+        "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz",
         "sysctl -n machdep.cpu.vendor": 'GenuineIntel',
     }
     if cmd in known_cmds:
@@ -210,6 +208,25 @@ def test_cpu_model(self):
         cpu_model = get_cpu_model()
         self.assertTrue(isinstance(cpu_model, basestring))
 
+        st.get_os_type = lambda: st.LINUX
+        st.read_file = mocked_read_file
+        st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
+        global PROC_CPUINFO_TXT
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
+        self.assertEqual(get_cpu_model(), "Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz")
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
+        self.assertEqual(get_cpu_model(), "IBM,8205-E6C")
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
+        self.assertEqual(get_cpu_model(), "ARMv7 Processor rev 5 (v7l)")
+
+        st.os.path.exists = self.orig_os_path_exists
+        st.get_os_type = lambda: st.DARWIN
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz")
+
     def test_cpu_speed(self):
         """Test getting CPU speed."""
         cpu_speed = get_cpu_speed()

From d6b33a0eedaa9e19d0cd1c3a25aad9d2b05e0a94 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 10:47:10 +0100
Subject: [PATCH 08/21] enhance error message in get_glibc_version

---
 easybuild/tools/systemtools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 52d8525ace..e89bffd809 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -398,7 +398,7 @@ def get_glibc_version():
             return glibc_version
         else:
             tup = (glibc_ver_str, glibc_ver_regex.pattern)
-            _log.error("Failed to determine version from '%s' using pattern '%s'." % tup)
+            _log.error("Failed to determine glibc version from '%s' using pattern '%s'." % tup)
     else:
         # no glibc on OS X standard
         _log.debug("No glibc on a non-Linux system, so can't determine version.")

From 708de3eb9bf54b2b0b37eaaf272b3975c64a1a28 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 10:59:03 +0100
Subject: [PATCH 09/21] moar systemtools testing via mocking used functions

---
 test/framework/systemtools.py | 49 +++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index 091e19309b..d0567fafb9 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -27,7 +27,7 @@
 
 @author: Kenneth hoste (Ghent University)
 """
-import os
+import re
 from os.path import exists as orig_os_path_exists
 from test.framework.utilities import EnhancedTestCase
 from unittest import TestLoader, main
@@ -36,7 +36,7 @@
 from easybuild.tools.filetools import read_file
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN
-from easybuild.tools.systemtools import get_avail_core_count
+from easybuild.tools.systemtools import det_parallelism, get_avail_core_count
 from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version
 from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext
 from easybuild.tools.systemtools import get_system_info
@@ -153,10 +153,12 @@ def mocked_os_path_exists(mocked_fp, fp):
 def mocked_run_cmd(cmd, **kwargs):
     """Mocked version of run_cmd, with specified output for known commands."""
     known_cmds = {
+        "ldd --version" : "ldd (GNU libc) 2.12",
         "sysctl -n hw.cpufrequency_max": "2400000000",
         "sysctl -n hw.ncpu": '10',
         "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz",
         "sysctl -n machdep.cpu.vendor": 'GenuineIntel',
+        "ulimit -u": '40',
     }
     if cmd in known_cmds:
         if 'simple' in kwargs and kwargs['simple']:
@@ -290,6 +292,12 @@ def test_shared_lib_ext(self):
         ext = get_shared_lib_ext()
         self.assertTrue(ext in ['dylib', 'so'])
 
+        st.get_os_type = lambda: st.LINUX
+        self.assertEqual(get_shared_lib_ext(), 'so')
+
+        st.get_os_type = lambda: st.DARWIN
+        self.assertEqual(get_shared_lib_ext(), 'dylib')
+
     def test_platform_name(self):
         """Test getting platform name."""
         platform_name_nover = get_platform_name()
@@ -303,6 +311,12 @@ def test_platform_name(self):
         self.assertTrue(platform_name_ver.startswith(platform_name_ver))
         self.assertTrue(len_ver >= len_nover)
 
+        st.get_os_type = lambda: st.LINUX
+        self.assertTrue(re.match('.*-unknown-linux$', get_platform_name()))
+
+        st.get_os_type = lambda: st.DARWIN
+        self.assertTrue(re.match('.*-apple-darwin$', get_platform_name()))
+
     def test_os_name(self):
         """Test getting OS name."""
         os_name = get_os_name()
@@ -318,11 +332,42 @@ def test_glibc_version(self):
         glibc_version = get_glibc_version()
         self.assertTrue(isinstance(glibc_version, basestring) or glibc_version == UNKNOWN)
 
+        st.get_os_type = lambda: st.LINUX
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_glibc_version(), '2.12')
+
+        st.get_os_type = lambda: st.DARWIN
+        self.assertEqual(get_glibc_version(), UNKNOWN)
+
     def test_system_info(self):
         """Test getting system info."""
         system_info = get_system_info()
         self.assertTrue(isinstance(system_info, dict))
 
+    def test_det_parallelism(self):
+        """Test det_parallelism function."""
+        self.assertTrue(det_parallelism(None, None) > 0)
+        # specified parallellism
+        self.assertEqual(det_parallelism(5, None), 5)
+        # max parallellism caps
+        self.assertEqual(det_parallelism(None, 1), 1)
+        self.assertEqual(det_parallelism(16, 1), 1)
+        self.assertEqual(det_parallelism(5, 2), 2)
+        self.assertEqual(det_parallelism(5, 10), 5)
+
+        orig_get_avail_core_count = st.get_avail_core_count
+
+        # mock number of available cores to 8
+        st.get_avail_core_count = lambda: 8
+        self.assertTrue(det_parallelism(None, None), 8)
+        # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6)
+        st.run_cmd = mocked_run_cmd
+        self.assertTrue(det_parallelism(None, None), 4)
+        self.assertTrue(det_parallelism(6, None), 4)
+        self.assertTrue(det_parallelism(2, None), 2)
+
+        st.get_avail_core_count = orig_get_avail_core_count
+
 def suite():
     """ returns all the testcases in this module """
     return TestLoader().loadTestsFromTestCase(SystemToolsTest)

From 1a9f9a3d02d329c74268ead145efc402a95f6413 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 11:50:44 +0100
Subject: [PATCH 10/21] add get_cpu_family() function in systemtools.py +
 dedicated test for it

---
 easybuild/tools/systemtools.py | 30 +++++++++++++++++++++++++++++-
 test/framework/systemtools.py  | 29 +++++++++++++++++++++++++++--
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index e89bffd809..00624b3e4d 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -48,7 +48,7 @@
 AMD = 'AMD'
 ARM = 'ARM'
 INTEL = 'Intel'
-PPC = 'PPC'
+POWER = 'POWER'
 
 LINUX = 'Linux'
 DARWIN = 'Darwin'
@@ -58,6 +58,7 @@
 MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq'
 PROC_CPUINFO_FP = '/proc/cpuinfo'
 
+CPU_FAMILIES = [ARM, AMD, INTEL, POWER]
 VENDORS = {
     'GenuineIntel': INTEL,
     'AuthenticAMD': AMD,
@@ -144,6 +145,33 @@ def get_cpu_vendor():
     return vendor
 
 
+def get_cpu_family():
+    """
+    Determine CPU family.
+    @return: one of the AMD, ARM, INTEL, POWER constants
+    """
+    family = None
+    vendor = get_cpu_vendor()
+    if vendor in CPU_FAMILIES:
+        family = vendor
+        _log.debug("Using vendor as CPU family: %s" % family)
+
+    else:
+        # POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo
+        if os.path.exists(PROC_CPUINFO_FP):
+            cpuinfo_txt = read_file(PROC_CPUINFO_FP)
+            power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M)
+            if power_regex.search(cpuinfo_txt):
+                family = POWER
+                tup = (power_regex.pattern, PROC_CPUINFO_FP, family)
+                _log.debug("Determined CPU family using regex '%s' in %s: %s" % tup)
+
+    if family is None:
+        family = UNKNOWN
+        _log.warning("Failed to determine CPU family, returning %s" % family)
+
+    return family
+
 def get_cpu_model():
     """
     Determine CPU model
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index d0567fafb9..d1d01beec5 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -35,8 +35,8 @@
 import easybuild.tools.systemtools as st
 from easybuild.tools.filetools import read_file
 from easybuild.tools.run import run_cmd
-from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN
-from easybuild.tools.systemtools import det_parallelism, get_avail_core_count
+from easybuild.tools.systemtools import CPU_FAMILIES, AMD, ARM, DARWIN, INTEL, LINUX, POWER, UNKNOWN
+from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family
 from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version
 from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext
 from easybuild.tools.systemtools import get_system_info
@@ -185,6 +185,7 @@ def tearDown(self):
         st.os.path.exists = self.orig_os_path_exists
         st.read_file = self.orig_read_file
         st.get_os_type = self.orig_get_os_type
+        st.run_cmd = self.orig_run_cmd
         super(SystemToolsTest, self).tearDown()
 
     def test_avail_core_count(self):
@@ -282,6 +283,30 @@ def test_cpu_vendor(self):
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_cpu_vendor(), INTEL)
 
+    def test_cpu_family(self):
+        """Test get_cpu_family function."""
+        cpu_family = get_cpu_family()
+        self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN)
+
+        st.get_os_type = lambda: st.LINUX
+        st.read_file = mocked_read_file
+        st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
+
+        global PROC_CPUINFO_TXT
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
+        self.assertEqual(get_cpu_family(), INTEL)
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
+        self.assertEqual(get_cpu_family(), ARM)
+
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
+        self.assertEqual(get_cpu_family(), POWER)
+
+        st.os.path.exists = self.orig_os_path_exists
+        st.get_os_type = lambda: st.DARWIN
+        st.run_cmd = mocked_run_cmd
+        self.assertEqual(get_cpu_family(), INTEL)
+
     def test_os_type(self):
         """Test getting OS type."""
         os_type = get_os_type()

From b6f57c3ad90ae87d5bf11a4570bca1372dcccbe4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 11:51:52 +0100
Subject: [PATCH 11/21] use get_cpu_family() rather than get_cpu_vendor() for
 picking optarch flag

---
 easybuild/tools/toolchain/compiler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py
index 9b72f621af..3946eca9ba 100644
--- a/easybuild/tools/toolchain/compiler.py
+++ b/easybuild/tools/toolchain/compiler.py
@@ -260,7 +260,7 @@ def _set_compiler_flags(self):
     def _get_optimal_architecture(self):
         """ Get options for the current architecture """
         if self.arch is None:
-            self.arch = systemtools.get_cpu_vendor()
+            self.arch = systemtools.get_cpu_family()
 
         optarch = None
         if build_option('optarch') is not None:
@@ -272,7 +272,7 @@ def _get_optimal_architecture(self):
             self.log.info("_get_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch))
             self.options.options_map['optarch'] = optarch
 
-        if 'optarch' in self.options.options_map and self.options.options_map.get('optarch', None) is None:
+        if self.options.options_map.get('optarch', None) is None:
             self.log.raiseException("_get_optimal_architecture: don't know how to set optarch for %s." % self.arch)
 
     def comp_family(self, prefix=None):

From c977b3e71502759f7d1680c8578387de424d444c Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 11:52:17 +0100
Subject: [PATCH 12/21] use -mcpu=native for GCC/Clang on POWER

---
 easybuild/toolchains/compiler/clang.py | 56 +++++++++++++-------------
 easybuild/toolchains/compiler/gcc.py   | 43 ++++++++++----------
 test/framework/toolchain.py            | 10 ++++-
 3 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py
index b78919bba1..d5b676bde1 100644
--- a/easybuild/toolchains/compiler/clang.py
+++ b/easybuild/toolchains/compiler/clang.py
@@ -53,38 +53,38 @@ class Clang(Compiler):
         'loop-vectorize': ['fvectorize'],
         'basic-block-vectorize': ['fslp-vectorize'],
         'optarch':'march=native',
-
-    # Clang's options do not map well onto these precision modes.  The flags enable and disable certain classes of
-    # optimizations.
-    # 
-    # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the
-    # ISO C and C++ language standard by possibly changing computation result.
-    # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division.
-    # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant.
-    # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs.
-    # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs.
-    # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results
-    # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities)
-    # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros,
-    # -freciprocal-math).
-    # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro
-    # __FAST_MATH__.
-    #
-    # Using -fno-fast-math is equivalent to disabling all individual optimizations, see
-    # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following)
-    #
-    # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details
-    # flags for strict and precise for robustness against future changes.
-    'strict': ['fno-fast-math'],
-    'precise': ['fno-unsafe-math-optimizations'],
-    'defaultprec': [],
-    'loose': ['ffast-math', 'fno-unsafe-math-optimizations'],
-    'veryloose': ['ffast-math'],
+        # Clang's options do not map well onto these precision modes.  The flags enable and disable certain classes of
+        # optimizations.
+        # 
+        # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the
+        # ISO C and C++ language standard by possibly changing computation result.
+        # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division.
+        # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant.
+        # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs.
+        # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs.
+        # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results
+        # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities)
+        # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros,
+        # -freciprocal-math).
+        # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro
+        # __FAST_MATH__.
+        #
+        # Using -fno-fast-math is equivalent to disabling all individual optimizations, see
+        # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following)
+        #
+        # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details
+        # flags for strict and precise for robustness against future changes.
+        'strict': ['fno-fast-math'],
+        'precise': ['fno-unsafe-math-optimizations'],
+        'defaultprec': [],
+        'loose': ['ffast-math', 'fno-unsafe-math-optimizations'],
+        'veryloose': ['ffast-math'],
     }
 
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
         systemtools.INTEL : 'march=native',
-        systemtools.AMD : 'march=native'
+        systemtools.AMD : 'march=native',
+        systemtools.POWER: 'mcpu=native',  # no support for march=native on POWER
     }
 
     COMPILER_CC = 'clang'
diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py
index e47a9c4778..3ec1888438 100644
--- a/easybuild/toolchains/compiler/gcc.py
+++ b/easybuild/toolchains/compiler/gcc.py
@@ -43,31 +43,30 @@ class Gcc(Compiler):
 
     COMPILER_FAMILY = TC_CONSTANT_GCC
     COMPILER_UNIQUE_OPTS = {
-                            'loop': (False, "Automatic loop parallellisation"),
-                            'f2c': (False, "Generate code compatible with f2c and f77"),
-                            'lto':(False, "Enable Link Time Optimization"),
-                            }
+        'loop': (False, "Automatic loop parallellisation"),
+        'f2c': (False, "Generate code compatible with f2c and f77"),
+        'lto':(False, "Enable Link Time Optimization"),
+    }
     COMPILER_UNIQUE_OPTION_MAP = {
-                                  'i8': 'fdefault-integer-8',
-                                  'r8': 'fdefault-real-8',
-                                  'unroll': 'funroll-loops',
-                                  'f2c': 'ff2c',
-                                  'loop': ['ftree-switch-conversion', 'floop-interchange',
-                                            'floop-strip-mine', 'floop-block'],
-                                  'lto':'flto',
-                                  'optarch':'march=native',
-                                  'openmp':'fopenmp',
-                                  'strict': ['mieee-fp', 'mno-recip'],
-                                  'precise':['mno-recip'],
-                                  'defaultprec':[],
-                                  'loose': ['mrecip', 'mno-ieee-fp'],
-                                  'veryloose': ['mrecip=all', 'mno-ieee-fp'],
-                                  }
+        'i8': 'fdefault-integer-8',
+        'r8': 'fdefault-real-8',
+        'unroll': 'funroll-loops',
+        'f2c': 'ff2c',
+        'loop': ['ftree-switch-conversion', 'floop-interchange', 'floop-strip-mine', 'floop-block'],
+        'lto': 'flto',
+        'openmp': 'fopenmp',
+        'strict': ['mieee-fp', 'mno-recip'],
+        'precise':['mno-recip'],
+        'defaultprec':[],
+        'loose': ['mrecip', 'mno-ieee-fp'],
+        'veryloose': ['mrecip=all', 'mno-ieee-fp'],
+    }
 
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
-                                            systemtools.INTEL : 'march=native',
-                                            systemtools.AMD : 'march=native'
-                                           }
+        systemtools.AMD : 'march=native',
+        systemtools.INTEL : 'march=native',
+        systemtools.POWER: 'mcpu=native',  # no support for march=native on POWER
+    }
 
     COMPILER_CC = 'gcc'
     COMPILER_CXX = 'g++'
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index dc60b2c041..fe1c238412 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -42,6 +42,10 @@
 from easybuild.tools.toolchain.utilities import search_toolchain
 from test.framework.utilities import find_full_path
 
+from easybuild.tools import systemtools as st
+import easybuild.tools.toolchain.compiler
+easybuild.tools.toolchain.compiler.systemtools.get_compiler_family = lambda: st.POWER
+
 class ToolchainTest(EnhancedTestCase):
     """ Baseclass for toolchain testcases """
 
@@ -295,6 +299,7 @@ def test_misc_flags_unique(self):
 
     def test_override_optarch(self):
         """Test whether overriding the optarch flag works."""
+        print st.get_compiler_family()
         flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
         for optarch_var in ['march=lovelylovelysandybridge', None]:
             build_options = {'optarch': optarch_var}
@@ -307,7 +312,8 @@ def test_override_optarch(self):
                 if optarch_var is not None:
                     flag = '-%s' % optarch_var
                 else:
-                    flag = '-march=native'
+                    # default optarch flag
+                    flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch]
 
                 for var in flag_vars:
                     flags = tc.get_variable(var)
@@ -389,7 +395,7 @@ def test_goolfc(self):
         tc.prepare()
 
         nvcc_flags = r' '.join([
-            r'-Xcompiler="-O2 -march=native"',
+            r'-Xcompiler="-O2 -%s"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch],
             # the use of -lcudart in -Xlinker is a bit silly but hard to avoid
             r'-Xlinker=".* -lm -lrt -lcudart -lpthread"',
             r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]),

From 9bde497864a70c49a08c2589486f4a1d74349753 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 12:09:57 +0100
Subject: [PATCH 13/21] split up systemtools tests into smaller ones (native vs
 mocked)

---
 easybuild/tools/systemtools.py |  1 +
 test/framework/systemtools.py  | 61 +++++++++++++++++++++++++---------
 2 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 00624b3e4d..9d910675a6 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -172,6 +172,7 @@ def get_cpu_family():
 
     return family
 
+
 def get_cpu_model():
     """
     Determine CPU model
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index d1d01beec5..fdec5a05db 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -188,12 +188,14 @@ def tearDown(self):
         st.run_cmd = self.orig_run_cmd
         super(SystemToolsTest, self).tearDown()
 
-    def test_avail_core_count(self):
+    def test_avail_core_count_native(self):
         """Test getting core count."""
         core_count = get_avail_core_count()
         self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count)))
         self.assertTrue(core_count > 0, "core_count %d > 0" % core_count)
 
+    def test_avail_core_count_linux(self):
+        """Test getting core count (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         orig_sched_getaffinity = st.sched_getaffinity
         class MockedSchedGetaffinity(object):
@@ -202,15 +204,19 @@ class MockedSchedGetaffinity(object):
         self.assertEqual(get_avail_core_count(), 6)
         st.sched_getaffinity = orig_sched_getaffinity
 
+    def test_avail_core_count_darwin(self):
+        """Test getting core count (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_avail_core_count(), 10)
 
-    def test_cpu_model(self):
+    def test_cpu_model_native(self):
         """Test getting CPU model."""
         cpu_model = get_cpu_model()
         self.assertTrue(isinstance(cpu_model, basestring))
 
+    def test_cpu_model_linux(self):
+        """Test getting CPU model (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         st.read_file = mocked_read_file
         st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
@@ -225,17 +231,20 @@ def test_cpu_model(self):
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
         self.assertEqual(get_cpu_model(), "ARMv7 Processor rev 5 (v7l)")
 
-        st.os.path.exists = self.orig_os_path_exists
+    def test_cpu_model_darwin(self):
+        """Test getting CPU model (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz")
 
-    def test_cpu_speed(self):
+    def test_cpu_speed_native(self):
         """Test getting CPU speed."""
         cpu_speed = get_cpu_speed()
         self.assertTrue(isinstance(cpu_speed, float))
         self.assertTrue(cpu_speed > 0.0)
 
+    def test_cpu_speed_linux(self):
+        """Test getting CPU speed (mocked for Linux)."""
         # test for particular type of system by mocking used functions
         st.get_os_type = lambda: st.LINUX
         st.read_file = mocked_read_file
@@ -256,8 +265,8 @@ def test_cpu_speed(self):
         st.os.path.exists = lambda fp: mocked_os_path_exists(MAX_FREQ_FP, fp)
         self.assertEqual(get_cpu_speed(), 2850.0)
 
-        # OS X
-        st.os.path.exists = self.orig_os_path_exists
+    def test_cpu_speed_darwin(self):
+        """Test getting CPU speed (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_cpu_speed(), 2400.0)
@@ -267,6 +276,8 @@ def test_cpu_vendor(self):
         cpu_vendor = get_cpu_vendor()
         self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN])
 
+    def test_cpu_vendor_linux(self):
+        """Test getting CPU vendor (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         st.read_file = mocked_read_file
         st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
@@ -278,21 +289,24 @@ def test_cpu_vendor(self):
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
         self.assertEqual(get_cpu_vendor(), ARM)
 
-        st.os.path.exists = self.orig_os_path_exists
+    def test_cpu_vendor_darwin(self):
+        """Test getting CPU vendor (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_cpu_vendor(), INTEL)
 
-    def test_cpu_family(self):
+    def test_cpu_family_native(self):
         """Test get_cpu_family function."""
         cpu_family = get_cpu_family()
         self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN)
 
+    def test_cpu_family_linux(self):
+        """Test get_cpu_family function (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         st.read_file = mocked_read_file
         st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
-
         global PROC_CPUINFO_TXT
+
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
         self.assertEqual(get_cpu_family(), INTEL)
 
@@ -302,7 +316,8 @@ def test_cpu_family(self):
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
         self.assertEqual(get_cpu_family(), POWER)
 
-        st.os.path.exists = self.orig_os_path_exists
+    def test_cpu_family_darwin(self):
+        """Test get_cpu_family function (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_cpu_family(), INTEL)
@@ -312,18 +327,22 @@ def test_os_type(self):
         os_type = get_os_type()
         self.assertTrue(os_type in [DARWIN, LINUX])
 
-    def test_shared_lib_ext(self):
+    def test_shared_lib_ext_native(self):
         """Test getting extension for shared libraries."""
         ext = get_shared_lib_ext()
         self.assertTrue(ext in ['dylib', 'so'])
 
+    def test_shared_lib_ext_native(self):
+        """Test getting extension for shared libraries (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         self.assertEqual(get_shared_lib_ext(), 'so')
 
+    def test_shared_lib_ext_native(self):
+        """Test getting extension for shared libraries (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         self.assertEqual(get_shared_lib_ext(), 'dylib')
 
-    def test_platform_name(self):
+    def test_platform_name_native(self):
         """Test getting platform name."""
         platform_name_nover = get_platform_name()
         self.assertTrue(isinstance(platform_name_nover, basestring))
@@ -336,11 +355,17 @@ def test_platform_name(self):
         self.assertTrue(platform_name_ver.startswith(platform_name_ver))
         self.assertTrue(len_ver >= len_nover)
 
+    def test_platform_name_linux(self):
+        """Test getting platform name (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         self.assertTrue(re.match('.*-unknown-linux$', get_platform_name()))
+        self.assertTrue(re.match('.*-unknown-linux-gnu$', get_platform_name(withversion=True)))
 
+    def test_platform_name_darwin(self):
+        """Test getting platform name (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         self.assertTrue(re.match('.*-apple-darwin$', get_platform_name()))
+        self.assertTrue(re.match('.*-apple-darwin.*$', get_platform_name(withversion=True)))
 
     def test_os_name(self):
         """Test getting OS name."""
@@ -352,15 +377,19 @@ def test_os_version(self):
         os_version = get_os_version()
         self.assertTrue(isinstance(os_version, basestring) or os_version == UNKNOWN)
 
-    def test_glibc_version(self):
+    def test_glibc_version_native(self):
         """Test getting glibc version."""
         glibc_version = get_glibc_version()
         self.assertTrue(isinstance(glibc_version, basestring) or glibc_version == UNKNOWN)
 
+    def test_glibc_version_linux(self):
+        """Test getting glibc version (mocked for Linux)."""
         st.get_os_type = lambda: st.LINUX
         st.run_cmd = mocked_run_cmd
         self.assertEqual(get_glibc_version(), '2.12')
 
+    def test_glibc_version_darwin(self):
+        """Test getting glibc version (mocked for Darwin)."""
         st.get_os_type = lambda: st.DARWIN
         self.assertEqual(get_glibc_version(), UNKNOWN)
 
@@ -369,8 +398,8 @@ def test_system_info(self):
         system_info = get_system_info()
         self.assertTrue(isinstance(system_info, dict))
 
-    def test_det_parallelism(self):
-        """Test det_parallelism function."""
+    def test_det_parallelism_native(self):
+        """Test det_parallelism function (native calls)."""
         self.assertTrue(det_parallelism(None, None) > 0)
         # specified parallellism
         self.assertEqual(det_parallelism(5, None), 5)
@@ -380,6 +409,8 @@ def test_det_parallelism(self):
         self.assertEqual(det_parallelism(5, 2), 2)
         self.assertEqual(det_parallelism(5, 10), 5)
 
+    def test_det_parallelism_mocked(self):
+        """Test det_parallelism function (with mocked ulimit/get_avail_core_count)."""
         orig_get_avail_core_count = st.get_avail_core_count
 
         # mock number of available cores to 8

From 0f8b5839e81f217acb1b8164513e9854757df0aa Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 12:16:43 +0100
Subject: [PATCH 14/21] re-add code that shouldn't have been removed in
 compiler.py

---
 easybuild/tools/toolchain/compiler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py
index 3946eca9ba..fda05be513 100644
--- a/easybuild/tools/toolchain/compiler.py
+++ b/easybuild/tools/toolchain/compiler.py
@@ -272,7 +272,7 @@ def _get_optimal_architecture(self):
             self.log.info("_get_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch))
             self.options.options_map['optarch'] = optarch
 
-        if self.options.options_map.get('optarch', None) is None:
+        if 'optarch' in self.options.options_map and self.options.options_map.get('optarch', None) is None:
             self.log.raiseException("_get_optimal_architecture: don't know how to set optarch for %s." % self.arch)
 
     def comp_family(self, prefix=None):

From 297dea1641208a8999a4c55304a718878668f2c1 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 13:17:23 +0100
Subject: [PATCH 15/21] fix broken test

---
 test/framework/toolchain.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index b7d6571f29..e6e80ff5e4 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -288,7 +288,10 @@ def test_misc_flags_unique(self):
                 tc = self.get_toolchain("goalf", version="1.1.0-no-OFED")
                 tc.set_options({opt: enable})
                 tc.prepare()
-                flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt]
+                if opt == 'optarch':
+                    flag = '-%s' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch]
+                else:
+                    flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt]
                 for var in flag_vars:
                     flags = tc.get_variable(var)
                     if enable:
@@ -299,7 +302,6 @@ def test_misc_flags_unique(self):
 
     def test_override_optarch(self):
         """Test whether overriding the optarch flag works."""
-        print st.get_compiler_family()
         flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']
         for optarch_var in ['march=lovelylovelysandybridge', None]:
             build_options = {'optarch': optarch_var}

From 11d199b56d85d146e74625e39f754b87d3bc69a9 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 16:22:33 +0100
Subject: [PATCH 16/21] fix small remark

---
 easybuild/tools/systemtools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 9d910675a6..58d0adcef9 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -134,7 +134,7 @@ def get_cpu_vendor():
         cmd = "sysctl -n machdep.cpu.vendor"
         out, ec = run_cmd(cmd)
         out = out.strip()
-        if ec == 0 and out and out in VENDORS:
+        if ec == 0 and out in VENDORS:
             vendor = VENDORS[out]
             _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
 

From eab31539a108aaaaf97d322363223e0ab243a4f9 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 16:49:50 +0100
Subject: [PATCH 17/21] add support for letting get_cpu_vendor return IBM +
 code cleanup

---
 easybuild/tools/systemtools.py | 22 ++++++++++------------
 test/framework/systemtools.py  |  5 ++++-
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 58d0adcef9..1414044a57 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -47,6 +47,7 @@
 # constants
 AMD = 'AMD'
 ARM = 'ARM'
+IBM = 'IBM'
 INTEL = 'Intel'
 POWER = 'POWER'
 
@@ -60,8 +61,10 @@
 
 CPU_FAMILIES = [ARM, AMD, INTEL, POWER]
 VENDORS = {
-    'GenuineIntel': INTEL,
+    'ARM': ARM,
     'AuthenticAMD': AMD,
+    'GenuineIntel': INTEL,
+    'IBM': IBM,
 }
 
 
@@ -112,24 +115,19 @@ def get_cpu_vendor():
         txt = read_file(PROC_CPUINFO_FP)
         arch = UNKNOWN
 
-        # vendor_id might not be in the /proc/cpuinfo, so this might fail
-        vendor_regex = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M)
+        vendor_regexes = [
+            r"^vendor_id\s+:\s*(\S+)\s*$",  # Linux/x86
+            r".*:\s*(ARM|IBM).*$",  # Linux/ARM (e.g., Raspbian), Linux/POWER
+        ]
+        vendor_regex = re.compile('|'.join(vendor_regexes), re.M)
         res = vendor_regex.search(txt)
         if res:
-            arch = res.group('vendorid')
+            arch = res.group(1) or res.group(2)
         if arch in VENDORS:
             vendor = VENDORS[arch]
             tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
             _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup)
 
-        # embedded Linux on ARM behaves differently (e.g. Raspbian)
-        vendor_regex = re.compile(r".*:\s*(?P<vendorid>ARM\S+)\s*", re.M)
-        res = vendor_regex.search(txt)
-        if res:
-            vendor = ARM
-            tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
-            _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup)
-
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.vendor"
         out, ec = run_cmd(cmd)
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index fdec5a05db..fdebc3a118 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -35,7 +35,7 @@
 import easybuild.tools.systemtools as st
 from easybuild.tools.filetools import read_file
 from easybuild.tools.run import run_cmd
-from easybuild.tools.systemtools import CPU_FAMILIES, AMD, ARM, DARWIN, INTEL, LINUX, POWER, UNKNOWN
+from easybuild.tools.systemtools import CPU_FAMILIES, AMD, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN
 from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family
 from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version
 from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext
@@ -286,6 +286,9 @@ def test_cpu_vendor_linux(self):
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
         self.assertEqual(get_cpu_vendor(), INTEL)
 
+        PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
+        self.assertEqual(get_cpu_vendor(), IBM)
+
         PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
         self.assertEqual(get_cpu_vendor(), ARM)
 

From cdc875dd70a5b96a28c787def641a8234a76f84f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 17:31:55 +0100
Subject: [PATCH 18/21] fix test_cpu_vendors test

---
 test/framework/systemtools.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index fdebc3a118..faf6f14265 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -35,7 +35,7 @@
 import easybuild.tools.systemtools as st
 from easybuild.tools.filetools import read_file
 from easybuild.tools.run import run_cmd
-from easybuild.tools.systemtools import CPU_FAMILIES, AMD, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN
+from easybuild.tools.systemtools import CPU_FAMILIES, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN, VENDORS
 from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family
 from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version
 from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext
@@ -274,7 +274,7 @@ def test_cpu_speed_darwin(self):
     def test_cpu_vendor(self):
         """Test getting CPU vendor."""
         cpu_vendor = get_cpu_vendor()
-        self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN])
+        self.assertTrue(cpu_vendor in VENDORS.values() + [UNKNOWN])
 
     def test_cpu_vendor_linux(self):
         """Test getting CPU vendor (mocked for Linux)."""

From 403acbaa3726d82ec667c45153457520493f3304 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 25 Feb 2015 17:35:11 +0100
Subject: [PATCH 19/21] fix mocked version of os.path.exists in systemtools
 tests, only return True for specified path

---
 test/framework/systemtools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py
index faf6f14265..c33319e96a 100644
--- a/test/framework/systemtools.py
+++ b/test/framework/systemtools.py
@@ -148,7 +148,7 @@ def mocked_read_file(fp):
 
 def mocked_os_path_exists(mocked_fp, fp):
     """Mocked version of os.path.exists, returns True for a particular specified filepath."""
-    return fp == mocked_fp or orig_os_path_exists(fp)
+    return fp == mocked_fp
 
 def mocked_run_cmd(cmd, **kwargs):
     """Mocked version of run_cmd, with specified output for known commands."""

From 06ec32a53617738f1bca670ca74efd38a3ede8d5 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 26 Feb 2015 11:24:22 +0100
Subject: [PATCH 20/21] fix remarks in systemtools.py

---
 easybuild/tools/systemtools.py | 31 +++++++++++--------------------
 1 file changed, 11 insertions(+), 20 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 1414044a57..4f0d52a7be 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -106,7 +106,7 @@ def get_cpu_vendor():
     """
     Try to detect the CPU vendor
 
-    @return: INTEL, ARM or AMD constant
+    @return: a value from the VENDORS dict
     """
     vendor = None
     os_type = get_os_type()
@@ -115,14 +115,10 @@ def get_cpu_vendor():
         txt = read_file(PROC_CPUINFO_FP)
         arch = UNKNOWN
 
-        vendor_regexes = [
-            r"^vendor_id\s+:\s*(\S+)\s*$",  # Linux/x86
-            r".*:\s*(ARM|IBM).*$",  # Linux/ARM (e.g., Raspbian), Linux/POWER
-        ]
-        vendor_regex = re.compile('|'.join(vendor_regexes), re.M)
+        vendor_regex = re.compile(r"(vendor_id.*?)?\s*:\s*(?P<vendor>(?(1)\S+|(?:IBM|ARM)))")
         res = vendor_regex.search(txt)
         if res:
-            arch = res.group(1) or res.group(2)
+            arch = res.group('vendor')
         if arch in VENDORS:
             vendor = VENDORS[arch]
             tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
@@ -146,7 +142,7 @@ def get_cpu_vendor():
 def get_cpu_family():
     """
     Determine CPU family.
-    @return: one of the AMD, ARM, INTEL, POWER constants
+    @return: a value from the CPU_FAMILIES list
     """
     family = None
     vendor = get_cpu_vendor()
@@ -173,14 +169,13 @@ def get_cpu_family():
 
 def get_cpu_model():
     """
-    Determine CPU model
-    for example: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
+    Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
     """
     model = None
     os_type = get_os_type()
 
     if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
-        # consider 'model name' first, use 'model' as a fallback
+        # we need 'model name' on Linux/x86, but 'model' is there first with different info
         # 'model name' is not there for Linux/POWER, but 'model' has the right info
         for key in [r'model\s*name', 'model']:
             model_regex = re.compile(r"^%s\s+:\s*(?P<model>.+)\s*$" % key, re.M)
@@ -225,16 +220,12 @@ def get_cpu_speed():
         elif os.path.exists(PROC_CPUINFO_FP):
             _log.debug("Trying to determine CPU frequency on Linux via %s" % PROC_CPUINFO_FP)
             cpuinfo_txt = read_file(PROC_CPUINFO_FP)
-            cpu_freq_regex = r"(%s)" % '|'.join([
-                r"^cpu MHz\s*:\s*(?P<cpu_freq_x86>[0-9.]+)",  # Linux x86 & more
-                r"^clock\s*:\s*(?P<cpu_freq_POWER>[0-9.]+)",  # Linux on POWER
-            ])
-            res = re.search(cpu_freq_regex, cpuinfo_txt, re.M)
+            # 'cpu MHz' on Linux/x86 (& more), 'clock' on Linux/POWER
+            cpu_freq_regex = re.compile(r"^(?:cpu MHz|clock)\s*:\s*(?P<cpu_freq>\d+(?:\.\d+)?)", re.M)
+            res = cpu_freq_regex.search(cpuinfo_txt)
             if res:
-                cpu_freq = res.group('cpu_freq_x86') or res.group('cpu_freq_POWER')
-            if cpu_freq is not None:
-                cpu_freq = float(cpu_freq)
-                _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex, cpu_freq))
+                cpu_freq = float(res.group('cpu_freq'))
+                _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex.pattern, cpu_freq))
             else:
                 raise SystemToolsException("Failed to determine CPU frequency from %s" % PROC_CPUINFO_FP)
         else:

From 7ff551129f090d8123f64f559d12dcf5897e6989 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 26 Feb 2015 12:54:15 +0100
Subject: [PATCH 21/21] use a single regex for obtaining model name

---
 easybuild/tools/systemtools.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 4f0d52a7be..921c8e926e 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -177,15 +177,13 @@ def get_cpu_model():
     if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
         # we need 'model name' on Linux/x86, but 'model' is there first with different info
         # 'model name' is not there for Linux/POWER, but 'model' has the right info
-        for key in [r'model\s*name', 'model']:
-            model_regex = re.compile(r"^%s\s+:\s*(?P<model>.+)\s*$" % key, re.M)
-            txt = read_file(PROC_CPUINFO_FP)
-            res = model_regex.search(txt)
-            if res is not None:
-                model = res.group('model').strip()
-                tup = (model_regex.pattern, PROC_CPUINFO_FP, model)
-                _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s" % tup)
-                break
+        model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M)
+        txt = read_file(PROC_CPUINFO_FP)
+        res = model_regex.search(txt)
+        if res is not None:
+            model = res.group('model').strip()
+            tup = (model_regex.pattern, PROC_CPUINFO_FP, model)
+            _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s" % tup)
 
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.brand_string"