diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 9397a9a584..ffbe45099c 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -96,7 +96,7 @@ def _set_compiler_flags(self): self.variables.nappend('CUDA_CXXFLAGS', cuda_flags) # add gencode compiler flags to list of flags for compiler variables - for gencode_val in self.options['cuda_gencode']: + for gencode_val in self.options.get('cuda_gencode', []): gencode_option = 'gencode %s' % gencode_val self.variables.nappend('CUDA_CFLAGS', gencode_option) self.variables.nappend('CUDA_CXXFLAGS', gencode_option) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 2e071033a2..e9af5dc722 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -87,7 +87,13 @@ class IntelIccIfort(Compiler): 'dynamic':'-Bdynamic', } - LIB_MULTITHREAD = ['iomp5', 'pthread'] ## iomp5 is OpenMP related + LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related + + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.append('LIB_MULTITHREAD') + super(IntelIccIfort, self).__init__(*args, **kwargs) def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 9594fa428a..fb7c5def68 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -58,6 +58,12 @@ class Acml(LinAlg): TC_CONSTANT_GCC: ['gfortran64', 'gfortran64_mp'], } + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB', 'BLAS_LIB_MT']) + super(Acml, self).__init__(*args, **kwargs) + def _set_blas_variables(self): """Fix the map a bit""" if self.options.get('32bit', None): diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 6d1886212a..81ae047c63 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -28,7 +28,6 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - from distutils.version import LooseVersion from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP @@ -69,11 +68,17 @@ class IntelMKL(LinAlg): SCALAPACK_MODULE_NAME = ['imkl'] SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] - SCALAPACK_LIB_MAP = {"lp64_sc":"_lp64"} + SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] SCALAPACK_LIB_GROUP = True SCALAPACK_LIB_STATIC = True + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) + super(IntelMKL, self).__init__(*args, **kwargs) + def _set_blas_variables(self): """Fix the map a bit""" interfacemap = { diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a5cad572be..907bb65734 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -30,7 +30,7 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +import copy import os from vsc.utils import fancylogger @@ -58,6 +58,10 @@ class Toolchain(object): VERSION = None TOOLCHAIN_FAMILY = None + # list of class 'constants' that should be restored for every new instance of this class + CLASS_CONSTANTS_TO_RESTORE = None + CLASS_CONSTANT_COPIES = {} + # class method def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" @@ -73,7 +77,7 @@ def _is_toolchain_for(cls, name): _is_toolchain_for = classmethod(_is_toolchain_for) - def __init__(self, name=None, version=None, mns=None): + def __init__(self, name=None, version=None, mns=None, class_constants=None): """Toolchain constructor.""" self.base_init() @@ -95,6 +99,8 @@ def __init__(self, name=None, version=None, mns=None): self.vars = None + self._init_class_constants(class_constants) + self.modules_tool = modules_tool() self.mns = mns self.mod_full_name = None @@ -109,6 +115,7 @@ def __init__(self, name=None, version=None, mns=None): self.init_modpaths = self.mns.det_init_modulepaths(tc_dict) def base_init(self): + """Initialise missing class attributes (log, options, variables).""" if not hasattr(self, 'log'): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -122,6 +129,43 @@ def base_init(self): if hasattr(self, 'LINKER_TOGGLE_STATIC_DYNAMIC'): self.variables.LINKER_TOGGLE_STATIC_DYNAMIC = self.LINKER_TOGGLE_STATIC_DYNAMIC + def _init_class_constants(self, class_constants): + """Initialise class 'constants'.""" + # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised + if class_constants is None: + self.CLASS_CONSTANTS_TO_RESTORE = [] + else: + self.CLASS_CONSTANTS_TO_RESTORE = class_constants[:] + + self._copy_class_constants() + self._restore_class_constants() + + def _copy_class_constants(self): + """Copy class constants that needs to be restored again when a new instance is created.""" + # this only needs to be done the first time (for this class, taking inheritance into account is key) + key = self.__class__ + if key not in self.CLASS_CONSTANT_COPIES: + self.CLASS_CONSTANT_COPIES[key] = {} + for cst in self.CLASS_CONSTANTS_TO_RESTORE: + if hasattr(self, cst): + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) + else: + raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + + self.log.debug("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) + + def _restore_class_constants(self): + """Restored class constants that need to be restored when a new instance is created.""" + key = self.__class__ + for cst in self.CLASS_CONSTANT_COPIES[key]: + newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) + if hasattr(self, cst): + self.log.debug("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) + else: + self.log.debug("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + + setattr(self, cst, newval) + def get_variable(self, name, typ=str): """Get value for specified variable. typ: indicates what type of return value is expected""" diff --git a/test/framework/modules.py b/test/framework/modules.py index 487cbb8ccb..aa201c095a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -46,7 +46,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 58 +TEST_MODULES_COUNT = 63 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/modules/icc/11.1.073 b/test/framework/modules/icc/11.1.073 new file mode 100644 index 0000000000..f78c11207f --- /dev/null +++ b/test/framework/modules/icc/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/icc/11.1.073 + +setenv EBROOTICC "$root" +setenv EBVERSIONICC "11.1.073" +setenv EBDEVELICC "$root/easybuild/icc-11.1.073-easybuild-devel" diff --git a/test/framework/modules/ictce/3.2.2.u3 b/test/framework/modules/ictce/3.2.2.u3 new file mode 100644 index 0000000000..453b686f4c --- /dev/null +++ b/test/framework/modules/ictce/3.2.2.u3 @@ -0,0 +1,36 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/ictce/3.2.2.u3 + +conflict ictce + +if { ![is-loaded icc/11.1.073] } { + module load icc/11.1.073 +} + +if { ![is-loaded ifort/11.1.073] } { + module load ifort/11.1.073 +} + +if { ![is-loaded impi/4.0.0.028] } { + module load impi/4.0.0.028 +} + +if { ![is-loaded imkl/10.2.6.038] } { + module load imkl/10.2.6.038 +} + + +setenv EBROOTICTCE "$root" +setenv EBVERSIONICTCE "3.2.2.u3" +setenv EBDEVELICTCE "$root/easybuild/ictce-3.2.2.u3-easybuild-devel" + + +# built with EasyBuild version 1.9.0dev diff --git a/test/framework/modules/ifort/11.1.073 b/test/framework/modules/ifort/11.1.073 new file mode 100644 index 0000000000..66e68eef4e --- /dev/null +++ b/test/framework/modules/ifort/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/ifort/11.1.073 + +setenv EBROOTIFORT "$root" +setenv EBVERSIONIFORT "11.1.073" +setenv EBDEVELIFORT "$root/easybuild/ifort-11.1.073-easybuild-devel" diff --git a/test/framework/modules/imkl/10.2.6.038 b/test/framework/modules/imkl/10.2.6.038 new file mode 100644 index 0000000000..c539b9e5f7 --- /dev/null +++ b/test/framework/modules/imkl/10.2.6.038 @@ -0,0 +1,7 @@ +#%Module + +set root /var/folders/8s/_frgh9sj6m744mxt5w5lyztr0000gn/T/eb-SGdeX9/tmptwWn9I + +setenv EBROOTIMKL "$root" +setenv EBVERSIONIMKL "10.2.6.038" +setenv EBDEVELIMKL "$root/easybuild/imkl-10.2.6.038-easybuild-devel" diff --git a/test/framework/modules/impi/4.0.0.028 b/test/framework/modules/impi/4.0.0.028 new file mode 100644 index 0000000000..ec11aaef29 --- /dev/null +++ b/test/framework/modules/impi/4.0.0.028 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/impi/4.0.0.028 + +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "4.0.0.028" +setenv EBDEVELIMPI "$root/easybuild/impi-4.0.0.028-easybuild-devel" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index e5f1d9890f..faf64fface 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -431,22 +431,22 @@ def test_goolfc(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) - def setup_sandbox_for_intel_fftw(self): + def setup_sandbox_for_intel_fftw(self, imklver='10.3.12.361'): """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass # rewrite $root in imkl module so we can put required lib*.a files in place tmpdir = tempfile.mkdtemp() test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) - imkl_module_path = os.path.join(test_modules_path, 'imkl', '10.3.12.361') + imkl_module_path = os.path.join(test_modules_path, 'imkl', imklver) imkl_module_txt = open(imkl_module_path, 'r').read() regex = re.compile('^(set\s*root).*$', re.M) imkl_module_alt_txt = regex.sub(r'\1\t%s' % tmpdir, imkl_module_txt) open(imkl_module_path, 'w').write(imkl_module_alt_txt) fftw_libs = ['fftw3xc_intel', 'fftw3x_cdft', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] - fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core'] - for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64']: + fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(tmpdir, subdir)) for fftlib in fftw_libs: write_file(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'foo') @@ -616,6 +616,77 @@ def test_prepare_deps_external(self): self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') self.assertEqual(modules.get_software_version('toy'), '1.2.3') + def test_old_new_iccifort(self): + """Test whether preparing for old/new Intel compilers works correctly.""" + tmpdir1, imkl_module_path1, imkl_module_txt1 = self.setup_sandbox_for_intel_fftw(imklver='10.3.12.361') + tmpdir2, imkl_module_path2, imkl_module_txt2 = self.setup_sandbox_for_intel_fftw(imklver='10.2.6.038') + + # incl. -lguide + libblas_mt_ictce3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" + + # no -lguide + libblas_mt_ictce4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce4 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + + # incl. -lmkl_solver* + libscalack_ictce3 = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64" + libscalack_ictce3 += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + # no -lmkl_solver* + libscalack_ictce4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + libblas_mt_goolfc = "-lopenblas -lgfortran" + libscalack_goolfc = "-lscalapack -lopenblas -lgfortran" + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + libscalack_ictce4 = libscalack_ictce4.replace('_lp64', '_ilp64') + tc = self.get_toolchain('ictce', version='4.1.13') + opts = {'i8': True} + tc.set_options(opts) + tc.prepare() + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) + + # cleanup + shutil.rmtree(tmpdir1) + shutil.rmtree(tmpdir2) + write_file(imkl_module_path1, imkl_module_txt1) + write_file(imkl_module_path2, imkl_module_txt2) def suite(): """ return all the tests"""