Merge pull request #1294 from boegel/keep_class_constants_constant

reset keep toolchain component class 'constants' every time

easybuild/toolchains/compiler/cuda.py

@@ -96,7 +96,7 @@ def _set_compiler_flags(self):
         self.variables.nappend('CUDA_CXXFLAGS', cuda_flags)
 
         # add gencode compiler flags to list of flags for compiler variables
-        for gencode_val in self.options['cuda_gencode']:
+        for gencode_val in self.options.get('cuda_gencode', []):
             gencode_option = 'gencode %s' % gencode_val
             self.variables.nappend('CUDA_CFLAGS', gencode_option)
             self.variables.nappend('CUDA_CXXFLAGS', gencode_option)

easybuild/toolchains/compiler/inteliccifort.py

@@ -87,7 +87,13 @@ class IntelIccIfort(Compiler):
         'dynamic':'-Bdynamic',
     }
 
-    LIB_MULTITHREAD = ['iomp5', 'pthread']  ## iomp5 is OpenMP related
+    LIB_MULTITHREAD = ['iomp5', 'pthread']  # iomp5 is OpenMP related
+
+    def __init__(self, *args, **kwargs):
+        """Toolchain constructor."""
+        class_constants = kwargs.setdefault('class_constants', [])
+        class_constants.append('LIB_MULTITHREAD')
+        super(IntelIccIfort, self).__init__(*args, **kwargs)
 
     def _set_compiler_vars(self):
         """Intel compilers-specific adjustments after setting compiler variables."""

easybuild/toolchains/linalg/acml.py

@@ -58,6 +58,12 @@ class Acml(LinAlg):
         TC_CONSTANT_GCC: ['gfortran64', 'gfortran64_mp'],
     }
 
+    def __init__(self, *args, **kwargs):
+        """Toolchain constructor."""
+        class_constants = kwargs.setdefault('class_constants', [])
+        class_constants.extend(['BLAS_LIB', 'BLAS_LIB_MT'])
+        super(Acml, self).__init__(*args, **kwargs)
+
     def _set_blas_variables(self):
         """Fix the map a bit"""
         if self.options.get('32bit', None):

easybuild/toolchains/linalg/intelmkl.py

@@ -28,7 +28,6 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-
 from distutils.version import LooseVersion
 
 from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP
@@ -69,11 +68,17 @@ class IntelMKL(LinAlg):
     SCALAPACK_MODULE_NAME = ['imkl']
     SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"]
     SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"]
-    SCALAPACK_LIB_MAP = {"lp64_sc":"_lp64"}
+    SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'}
     SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS']
     SCALAPACK_LIB_GROUP = True
     SCALAPACK_LIB_STATIC = True
 
+    def __init__(self, *args, **kwargs):
+        """Toolchain constructor."""
+        class_constants = kwargs.setdefault('class_constants', [])
+        class_constants.extend(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP'])
+        super(IntelMKL, self).__init__(*args, **kwargs)
+
     def _set_blas_variables(self):
         """Fix the map a bit"""
         interfacemap = {

easybuild/tools/toolchain/toolchain.py

@@ -30,7 +30,7 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-
+import copy
 import os
 from vsc.utils import fancylogger
 
@@ -58,6 +58,10 @@ class Toolchain(object):
     VERSION = None
     TOOLCHAIN_FAMILY = None
 
+    # list of class 'constants' that should be restored for every new instance of this class
+    CLASS_CONSTANTS_TO_RESTORE = None
+    CLASS_CONSTANT_COPIES = {}
+
     # class method
     def _is_toolchain_for(cls, name):
         """see if this class can provide support for toolchain named name"""
@@ -73,7 +77,7 @@ def _is_toolchain_for(cls, name):
 
     _is_toolchain_for = classmethod(_is_toolchain_for)
 
-    def __init__(self, name=None, version=None, mns=None):
+    def __init__(self, name=None, version=None, mns=None, class_constants=None):
         """Toolchain constructor."""
 
         self.base_init()
@@ -95,6 +99,8 @@ def __init__(self, name=None, version=None, mns=None):
 
         self.vars = None
 
+        self._init_class_constants(class_constants)
+
         self.modules_tool = modules_tool()
         self.mns = mns
         self.mod_full_name = None
@@ -109,6 +115,7 @@ def __init__(self, name=None, version=None, mns=None):
                 self.init_modpaths = self.mns.det_init_modulepaths(tc_dict)
 
     def base_init(self):
+        """Initialise missing class attributes (log, options, variables)."""
         if not hasattr(self, 'log'):
             self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
 
@@ -122,6 +129,43 @@ def base_init(self):
             if hasattr(self, 'LINKER_TOGGLE_STATIC_DYNAMIC'):
                 self.variables.LINKER_TOGGLE_STATIC_DYNAMIC = self.LINKER_TOGGLE_STATIC_DYNAMIC
 
+    def _init_class_constants(self, class_constants):
+        """Initialise class 'constants'."""
+        # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised
+        if class_constants is None:
+            self.CLASS_CONSTANTS_TO_RESTORE = []
+        else:
+            self.CLASS_CONSTANTS_TO_RESTORE = class_constants[:]
+
+        self._copy_class_constants()
+        self._restore_class_constants()
+
+    def _copy_class_constants(self):
+        """Copy class constants that needs to be restored again when a new instance is created."""
+        # this only needs to be done the first time (for this class, taking inheritance into account is key)
+        key = self.__class__
+        if key not in self.CLASS_CONSTANT_COPIES:
+            self.CLASS_CONSTANT_COPIES[key] = {}
+            for cst in self.CLASS_CONSTANTS_TO_RESTORE:
+                if hasattr(self, cst):
+                    self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst))
+                else:
+                    raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self)
+
+            self.log.debug("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key])
+
+    def _restore_class_constants(self):
+        """Restored class constants that need to be restored when a new instance is created."""
+        key = self.__class__
+        for cst in self.CLASS_CONSTANT_COPIES[key]:
+            newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst])
+            if hasattr(self, cst):
+                self.log.debug("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst))
+            else:
+                self.log.debug("Restoring (currently undefined) class constant '%s' to %s", cst, newval)
+
+            setattr(self, cst, newval)
+
     def get_variable(self, name, typ=str):
         """Get value for specified variable.
         typ: indicates what type of return value is expected"""

test/framework/modules.py

@@ -46,7 +46,7 @@
 
 
 # number of modules included for testing purposes
-TEST_MODULES_COUNT = 58
+TEST_MODULES_COUNT = 63
 
 
 class ModulesTest(EnhancedTestCase):

test/framework/modules/icc/11.1.073

@@ -0,0 +1,7 @@
+#%Module
+
+set root /tmp/icc/11.1.073
+
+setenv EBROOTICC "$root"
+setenv EBVERSIONICC "11.1.073"
+setenv  EBDEVELICC "$root/easybuild/icc-11.1.073-easybuild-devel"

test/framework/modules/ictce/3.2.2.u3

@@ -0,0 +1,36 @@
+#%Module
+
+proc ModulesHelp { } {
+    puts stderr {   Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/
+    }
+}
+
+module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/}
+
+set root    /tmp/ictce/3.2.2.u3
+
+conflict    ictce
+
+if { ![is-loaded icc/11.1.073] } {
+    module load icc/11.1.073
+}
+
+if { ![is-loaded ifort/11.1.073] } {
+    module load ifort/11.1.073
+}
+
+if { ![is-loaded impi/4.0.0.028] } {
+    module load impi/4.0.0.028
+}
+
+if { ![is-loaded imkl/10.2.6.038] } {
+    module load imkl/10.2.6.038
+}
+
+
+setenv	EBROOTICTCE		"$root"
+setenv	EBVERSIONICTCE		"3.2.2.u3"
+setenv	EBDEVELICTCE		"$root/easybuild/ictce-3.2.2.u3-easybuild-devel"
+
+
+# built with EasyBuild version 1.9.0dev

test/framework/modules/ifort/11.1.073

@@ -0,0 +1,7 @@
+#%Module
+
+set root /tmp/ifort/11.1.073
+
+setenv EBROOTIFORT "$root"
+setenv EBVERSIONIFORT "11.1.073"
+setenv  EBDEVELIFORT "$root/easybuild/ifort-11.1.073-easybuild-devel"

test/framework/modules/imkl/10.2.6.038

@@ -0,0 +1,7 @@
+#%Module
+
+set root	/var/folders/8s/_frgh9sj6m744mxt5w5lyztr0000gn/T/eb-SGdeX9/tmptwWn9I
+
+setenv EBROOTIMKL "$root"
+setenv EBVERSIONIMKL "10.2.6.038"
+setenv  EBDEVELIMKL "$root/easybuild/imkl-10.2.6.038-easybuild-devel"

test/framework/modules/impi/4.0.0.028

@@ -0,0 +1,7 @@
+#%Module
+
+set root /tmp/impi/4.0.0.028
+
+setenv EBROOTIMPI "$root"
+setenv EBVERSIONIMPI "4.0.0.028"
+setenv  EBDEVELIMPI "$root/easybuild/impi-4.0.0.028-easybuild-devel"

test/framework/toolchain.py

@@ -431,22 +431,22 @@ def test_goolfc(self):
         # check CUDA runtime lib
         self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS'))
 
-    def setup_sandbox_for_intel_fftw(self):
+    def setup_sandbox_for_intel_fftw(self, imklver='10.3.12.361'):
         """Set up sandbox for Intel FFTW"""
         # hack to make Intel FFTW lib check pass
         # rewrite $root in imkl module so we can put required lib*.a files in place
         tmpdir = tempfile.mkdtemp()
 
         test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
-        imkl_module_path = os.path.join(test_modules_path, 'imkl', '10.3.12.361')
+        imkl_module_path = os.path.join(test_modules_path, 'imkl', imklver)
         imkl_module_txt = open(imkl_module_path, 'r').read()
         regex = re.compile('^(set\s*root).*$', re.M)
         imkl_module_alt_txt = regex.sub(r'\1\t%s' % tmpdir, imkl_module_txt)
         open(imkl_module_path, 'w').write(imkl_module_alt_txt)
 
         fftw_libs = ['fftw3xc_intel', 'fftw3x_cdft', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64']
-        fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core']
-        for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64']:
+        fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64']
+        for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']:
             os.makedirs(os.path.join(tmpdir, subdir))
             for fftlib in fftw_libs:
                 write_file(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'foo')
@@ -616,6 +616,77 @@ def test_prepare_deps_external(self):
         self.assertEqual(modules.get_software_root('foobar'), '/foo/bar')
         self.assertEqual(modules.get_software_version('toy'), '1.2.3')
 
+    def test_old_new_iccifort(self):
+        """Test whether preparing for old/new Intel compilers works correctly."""
+        tmpdir1, imkl_module_path1, imkl_module_txt1 = self.setup_sandbox_for_intel_fftw(imklver='10.3.12.361')
+        tmpdir2, imkl_module_path2, imkl_module_txt2 = self.setup_sandbox_for_intel_fftw(imklver='10.2.6.038')
+
+        # incl. -lguide
+        libblas_mt_ictce3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core"
+        libblas_mt_ictce3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread"
+
+        # no -lguide
+        libblas_mt_ictce4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core"
+        libblas_mt_ictce4 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread"
+
+        # incl. -lmkl_solver*
+        libscalack_ictce3 = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64"
+        libscalack_ictce3 += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core"
+
+        # no -lmkl_solver*
+        libscalack_ictce4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core"
+
+        libblas_mt_goolfc = "-lopenblas -lgfortran"
+        libscalack_goolfc = "-lscalapack -lopenblas -lgfortran"
+
+        tc = self.get_toolchain('goolfc', version='1.3.12')
+        tc.prepare()
+        self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc)
+        self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc)
+        modules_tool().purge()
+
+        tc = self.get_toolchain('ictce', version='4.1.13')
+        tc.prepare()
+        self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4)
+        self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK'])
+        modules_tool().purge()
+
+        tc = self.get_toolchain('ictce', version='3.2.2.u3')
+        tc.prepare()
+        self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3)
+        self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK'])
+        modules_tool().purge()
+
+        tc = self.get_toolchain('ictce', version='4.1.13')
+        tc.prepare()
+        self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4)
+        self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK'])
+        modules_tool().purge()
+
+        tc = self.get_toolchain('ictce', version='3.2.2.u3')
+        tc.prepare()
+        self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3)
+        self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK'])
+        modules_tool().purge()
+
+        libscalack_ictce4 = libscalack_ictce4.replace('_lp64', '_ilp64')
+        tc = self.get_toolchain('ictce', version='4.1.13')
+        opts = {'i8': True}
+        tc.set_options(opts)
+        tc.prepare()
+        self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK'])
+        modules_tool().purge()
+
+        tc = self.get_toolchain('goolfc', version='1.3.12')
+        tc.prepare()
+        self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc)
+        self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc)
+
+        # cleanup
+        shutil.rmtree(tmpdir1)
+        shutil.rmtree(tmpdir2)
+        write_file(imkl_module_path1, imkl_module_txt1)
+        write_file(imkl_module_path2, imkl_module_txt2)
 
 def suite():
     """ return all the tests"""