From 02e85d9ce1516b022be870dc103916c00a738ab1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 09:08:16 +0200 Subject: [PATCH 01/10] reset keep toolchain component class 'constants' every time --- easybuild/toolchains/compiler/inteliccifort.py | 5 ++++- easybuild/toolchains/linalg/acml.py | 10 +++++++--- easybuild/toolchains/linalg/intelmkl.py | 15 ++++++++++----- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 2e071033a2..8214933f61 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -87,7 +87,7 @@ class IntelIccIfort(Compiler): 'dynamic':'-Bdynamic', } - LIB_MULTITHREAD = ['iomp5', 'pthread'] ## iomp5 is OpenMP related + LIB_MULTITHREAD = None def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" @@ -104,6 +104,9 @@ def _set_compiler_vars(self): raise EasyBuildError("_set_compiler_vars: mismatch between icc version %s and ifort version %s", icc_version, ifort_version) + # reset LIB_MULTITHREAD every time, + # to avoid problems when multiple Intel compilers versions are used in a single session + self.LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related if LooseVersion(icc_version) < LooseVersion('2011'): self.LIB_MULTITHREAD.insert(1, "guide") diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 9594fa428a..befed0aaf9 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -43,9 +43,8 @@ class Acml(LinAlg): Provides ACML BLAS/LAPACK support. """ BLAS_MODULE_NAME = ['ACML'] - # full list of libraries is highly dependent on ACML version and toolchain compiler (ifort, gfortran, ...) - BLAS_LIB = ['acml'] - BLAS_LIB_MT = ['acml_mp'] + BLAS_LIB = None + BLAS_LIB_MT = None # is completed in _set_blas_variables, depends on compiler used BLAS_LIB_DIR = [] @@ -73,6 +72,11 @@ def _set_blas_variables(self): raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " "with compiler family %s", self.COMPILER_FAMILY) + # reset BLAS_LIB(_MT) every time, to avoid problems when multiple ACML versions are used in a single session + # full list of libraries is highly dependent on ACML version and toolchain compiler (ifort, gfortran, ...) + self.BLAS_LIB = ['acml'] + self.BLAS_LIB_MT = ['acml_mp'] + # version before 5.x still featured the acml_mv library ver = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(ver) < LooseVersion("5"): diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 6d1886212a..2c86cc680b 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -67,9 +67,9 @@ class IntelMKL(LinAlg): BLACS_LIB_STATIC = True SCALAPACK_MODULE_NAME = ['imkl'] - SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] - SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] - SCALAPACK_LIB_MAP = {"lp64_sc":"_lp64"} + SCALAPACK_LIB = None + SCALAPACK_LIB_MT = None + SCALAPACK_LIB_MAP = None SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] SCALAPACK_LIB_GROUP = True SCALAPACK_LIB_STATIC = True @@ -101,10 +101,10 @@ def _set_blas_variables(self): if self.options.get('32bit', None): # 32bit - self.BLAS_LIB_MAP.update({"lp64":''}) + self.BLAS_LIB_MAP.update({"lp64":''}) # FIXME if self.options.get('i8', None): # ilp64/i8 - self.BLAS_LIB_MAP.update({"lp64":'_ilp64'}) + self.BLAS_LIB_MAP.update({"lp64":'_ilp64'}) # FIXME # CPP / CFLAGS self.variables.nappend_el('CFLAGS', 'DMKL_ILP64') @@ -150,6 +150,11 @@ def _set_blacs_variables(self): super(IntelMKL, self)._set_blacs_variables() def _set_scalapack_variables(self): + # reset SCALAPACK_LIB* every time, to avoid problems when multiple imkl versions are used in a single session + self.SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] + self.SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] + self.SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} + imkl_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(imkl_version) < LooseVersion('10.3'): self.SCALAPACK_LIB.append("mkl_solver%(lp64)s_sequential") From f532b8d6cd95580e9e86fa06630fe7157623134e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 09:48:35 +0200 Subject: [PATCH 02/10] add unit test to verify fix w.r.t. toolchain constants --- test/framework/modules/icc/11.1.073 | 7 +++++ test/framework/modules/ictce/3.2.2.u3 | 36 +++++++++++++++++++++ test/framework/modules/ifort/11.1.073 | 7 +++++ test/framework/modules/imkl/10.2.6.038 | 7 +++++ test/framework/modules/impi/4.0.0.028 | 7 +++++ test/framework/toolchain.py | 43 ++++++++++++++++++++++++-- 6 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 test/framework/modules/icc/11.1.073 create mode 100644 test/framework/modules/ictce/3.2.2.u3 create mode 100644 test/framework/modules/ifort/11.1.073 create mode 100644 test/framework/modules/imkl/10.2.6.038 create mode 100644 test/framework/modules/impi/4.0.0.028 diff --git a/test/framework/modules/icc/11.1.073 b/test/framework/modules/icc/11.1.073 new file mode 100644 index 0000000000..f78c11207f --- /dev/null +++ b/test/framework/modules/icc/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/icc/11.1.073 + +setenv EBROOTICC "$root" +setenv EBVERSIONICC "11.1.073" +setenv EBDEVELICC "$root/easybuild/icc-11.1.073-easybuild-devel" diff --git a/test/framework/modules/ictce/3.2.2.u3 b/test/framework/modules/ictce/3.2.2.u3 new file mode 100644 index 0000000000..453b686f4c --- /dev/null +++ b/test/framework/modules/ictce/3.2.2.u3 @@ -0,0 +1,36 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/ictce/3.2.2.u3 + +conflict ictce + +if { ![is-loaded icc/11.1.073] } { + module load icc/11.1.073 +} + +if { ![is-loaded ifort/11.1.073] } { + module load ifort/11.1.073 +} + +if { ![is-loaded impi/4.0.0.028] } { + module load impi/4.0.0.028 +} + +if { ![is-loaded imkl/10.2.6.038] } { + module load imkl/10.2.6.038 +} + + +setenv EBROOTICTCE "$root" +setenv EBVERSIONICTCE "3.2.2.u3" +setenv EBDEVELICTCE "$root/easybuild/ictce-3.2.2.u3-easybuild-devel" + + +# built with EasyBuild version 1.9.0dev diff --git a/test/framework/modules/ifort/11.1.073 b/test/framework/modules/ifort/11.1.073 new file mode 100644 index 0000000000..66e68eef4e --- /dev/null +++ b/test/framework/modules/ifort/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/ifort/11.1.073 + +setenv EBROOTIFORT "$root" +setenv EBVERSIONIFORT "11.1.073" +setenv EBDEVELIFORT "$root/easybuild/ifort-11.1.073-easybuild-devel" diff --git a/test/framework/modules/imkl/10.2.6.038 b/test/framework/modules/imkl/10.2.6.038 new file mode 100644 index 0000000000..c539b9e5f7 --- /dev/null +++ b/test/framework/modules/imkl/10.2.6.038 @@ -0,0 +1,7 @@ +#%Module + +set root /var/folders/8s/_frgh9sj6m744mxt5w5lyztr0000gn/T/eb-SGdeX9/tmptwWn9I + +setenv EBROOTIMKL "$root" +setenv EBVERSIONIMKL "10.2.6.038" +setenv EBDEVELIMKL "$root/easybuild/imkl-10.2.6.038-easybuild-devel" diff --git a/test/framework/modules/impi/4.0.0.028 b/test/framework/modules/impi/4.0.0.028 new file mode 100644 index 0000000000..ec11aaef29 --- /dev/null +++ b/test/framework/modules/impi/4.0.0.028 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/impi/4.0.0.028 + +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "4.0.0.028" +setenv EBDEVELIMPI "$root/easybuild/impi-4.0.0.028-easybuild-devel" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index e5f1d9890f..f37a3c2870 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -431,14 +431,14 @@ def test_goolfc(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) - def setup_sandbox_for_intel_fftw(self): + def setup_sandbox_for_intel_fftw(self, imklver='10.3.12.361'): """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass # rewrite $root in imkl module so we can put required lib*.a files in place tmpdir = tempfile.mkdtemp() test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) - imkl_module_path = os.path.join(test_modules_path, 'imkl', '10.3.12.361') + imkl_module_path = os.path.join(test_modules_path, 'imkl', imklver) imkl_module_txt = open(imkl_module_path, 'r').read() regex = re.compile('^(set\s*root).*$', re.M) imkl_module_alt_txt = regex.sub(r'\1\t%s' % tmpdir, imkl_module_txt) @@ -446,7 +446,7 @@ def setup_sandbox_for_intel_fftw(self): fftw_libs = ['fftw3xc_intel', 'fftw3x_cdft', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core'] - for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64']: + for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(tmpdir, subdir)) for fftlib in fftw_libs: write_file(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'foo') @@ -616,6 +616,43 @@ def test_prepare_deps_external(self): self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') self.assertEqual(modules.get_software_version('toy'), '1.2.3') + def test_old_new_iccifort(self): + """Test whether preparing for old/new Intel compilers works correctly.""" + tmpdir1, imkl_module_path1, imkl_module_txt1 = self.setup_sandbox_for_intel_fftw(imklver='10.3.12.361') + tmpdir2, imkl_module_path2, imkl_module_txt2 = self.setup_sandbox_for_intel_fftw(imklver='10.2.6.038') + + # incl. -lguide + libblas_mt_old = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_old += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" + + # no -lguide + libblas_mt_new = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_new += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) + + # cleanup + shutil.rmtree(tmpdir1) + shutil.rmtree(tmpdir2) + write_file(imkl_module_path1, imkl_module_txt1) + write_file(imkl_module_path2, imkl_module_txt2) def suite(): """ return all the tests""" From 2cee2ad8a1b9ecf0255b5833bf269a74afa7a94f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 10:28:42 +0200 Subject: [PATCH 03/10] fix modules count in modules.py test module --- test/framework/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 487cbb8ccb..aa201c095a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -46,7 +46,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 58 +TEST_MODULES_COUNT = 63 class ModulesTest(EnhancedTestCase): From 5c94dd18f7ef71412fdc15678597b071736d6135 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 11:01:17 +0200 Subject: [PATCH 04/10] make fix inheritance-safe --- easybuild/toolchains/compiler/inteliccifort.py | 6 ++++-- easybuild/toolchains/linalg/acml.py | 11 +++++++---- easybuild/toolchains/linalg/intelmkl.py | 16 ++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 8214933f61..7532468998 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -87,7 +87,9 @@ class IntelIccIfort(Compiler): 'dynamic':'-Bdynamic', } - LIB_MULTITHREAD = None + LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related + # keep track of original value, needs to be restored every time since we append to class 'constant' LIB_MULTITHREAD + _INIT_LIB_MULTITHREAD = LIB_MULTITHREAD[:] def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" @@ -106,7 +108,7 @@ def _set_compiler_vars(self): # reset LIB_MULTITHREAD every time, # to avoid problems when multiple Intel compilers versions are used in a single session - self.LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related + self.LIB_MULTITHREAD = self._INIT_LIB_MULTITHREAD[:] if LooseVersion(icc_version) < LooseVersion('2011'): self.LIB_MULTITHREAD.insert(1, "guide") diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index befed0aaf9..13876777f3 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -43,8 +43,11 @@ class Acml(LinAlg): Provides ACML BLAS/LAPACK support. """ BLAS_MODULE_NAME = ['ACML'] - BLAS_LIB = None - BLAS_LIB_MT = None + BLAS_LIB = ['acml'] + BLAS_LIB_MT = ['acml_mp'] + # keep track of original values, need to be restored every time since we append to class 'constants' BLAS_LIB* + _INIT_BLAS_LIB = BLAS_LIB[:] + _INIT_BLAS_LIB_MT = BLAS_LIB_MT[:] # is completed in _set_blas_variables, depends on compiler used BLAS_LIB_DIR = [] @@ -74,8 +77,8 @@ def _set_blas_variables(self): # reset BLAS_LIB(_MT) every time, to avoid problems when multiple ACML versions are used in a single session # full list of libraries is highly dependent on ACML version and toolchain compiler (ifort, gfortran, ...) - self.BLAS_LIB = ['acml'] - self.BLAS_LIB_MT = ['acml_mp'] + self.BLAS_LIB = self._INIT_BLAS_LIB[:] + self.BLAS_LIB_MT = self._INIT_BLAS_LIB_MT[:] # version before 5.x still featured the acml_mv library ver = self.get_software_version(self.BLAS_MODULE_NAME)[0] diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 2c86cc680b..2cc5cb663e 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -67,12 +67,16 @@ class IntelMKL(LinAlg): BLACS_LIB_STATIC = True SCALAPACK_MODULE_NAME = ['imkl'] - SCALAPACK_LIB = None - SCALAPACK_LIB_MT = None - SCALAPACK_LIB_MAP = None SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] SCALAPACK_LIB_GROUP = True SCALAPACK_LIB_STATIC = True + SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] + SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] + SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} + # keep track of original values, need to be restored every time since we append to class 'constants' SCALAPACK_LIB* + _INIT_SCALAPACK_LIB = SCALAPACK_LIB[:] + _INIT_SCALAPACK_LIB_MT = SCALAPACK_LIB_MT[:] + _INIT_SCALAPACK_LIB_MAP = SCALAPACK_LIB_MAP[:] def _set_blas_variables(self): """Fix the map a bit""" @@ -151,9 +155,9 @@ def _set_blacs_variables(self): def _set_scalapack_variables(self): # reset SCALAPACK_LIB* every time, to avoid problems when multiple imkl versions are used in a single session - self.SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] - self.SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] - self.SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} + self.SCALAPACK_LIB = self._INIT_SCALAPACK_LIB + self.SCALAPACK_LIB_MT = self._INIT_SCALAPACK_LIB_MT + self.SCALAPACK_LIB_MAP = self._INIT_SCALAPACK_LIB_MAP imkl_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(imkl_version) < LooseVersion('10.3'): From 8d1cdf425d824fc5560f4a2fb596624895ee45aa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 11:17:34 +0200 Subject: [PATCH 05/10] fix resetting of self.SCALAPACK_LIB_MAP, enhance test --- easybuild/toolchains/linalg/intelmkl.py | 10 +++++----- test/framework/toolchain.py | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 2cc5cb663e..ae3a3e7914 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -28,7 +28,7 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +import copy from distutils.version import LooseVersion from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP @@ -76,7 +76,7 @@ class IntelMKL(LinAlg): # keep track of original values, need to be restored every time since we append to class 'constants' SCALAPACK_LIB* _INIT_SCALAPACK_LIB = SCALAPACK_LIB[:] _INIT_SCALAPACK_LIB_MT = SCALAPACK_LIB_MT[:] - _INIT_SCALAPACK_LIB_MAP = SCALAPACK_LIB_MAP[:] + _INIT_SCALAPACK_LIB_MAP = copy.deepcopy(SCALAPACK_LIB_MAP) def _set_blas_variables(self): """Fix the map a bit""" @@ -155,9 +155,9 @@ def _set_blacs_variables(self): def _set_scalapack_variables(self): # reset SCALAPACK_LIB* every time, to avoid problems when multiple imkl versions are used in a single session - self.SCALAPACK_LIB = self._INIT_SCALAPACK_LIB - self.SCALAPACK_LIB_MT = self._INIT_SCALAPACK_LIB_MT - self.SCALAPACK_LIB_MAP = self._INIT_SCALAPACK_LIB_MAP + self.SCALAPACK_LIB = self._INIT_SCALAPACK_LIB[:] + self.SCALAPACK_LIB_MT = self._INIT_SCALAPACK_LIB_MT[:] + self.SCALAPACK_LIB_MAP = copy.deepcopy(self._INIT_SCALAPACK_LIB_MAP) imkl_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(imkl_version) < LooseVersion('10.3'): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f37a3c2870..7fee7d0831 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -445,7 +445,7 @@ def setup_sandbox_for_intel_fftw(self, imklver='10.3.12.361'): open(imkl_module_path, 'w').write(imkl_module_alt_txt) fftw_libs = ['fftw3xc_intel', 'fftw3x_cdft', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] - fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core'] + fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(tmpdir, subdir)) for fftlib in fftw_libs: @@ -629,24 +629,43 @@ def test_old_new_iccifort(self): libblas_mt_new = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" libblas_mt_new += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + # incl. -lmkl_solver* + libscalack_old = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64" + libscalack_old += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + # no -lmkl_solver* + libscalack_new = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + tc = self.get_toolchain('ictce', version='4.1.13') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) + self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='3.2.2.u3') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) + self.assertTrue(libscalack_old in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='4.1.13') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) + self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='3.2.2.u3') tc.prepare() self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) + self.assertTrue(libscalack_old in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + libscalack_new = libscalack_new.replace('_lp64', '_ilp64') + tc = self.get_toolchain('ictce', version='4.1.13') + opts = {'i8': True} + tc.set_options(opts) + tc.prepare() + self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) # cleanup shutil.rmtree(tmpdir1) From 391cc62e6052425c580102925d6e5e975711a398 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 16:54:30 +0200 Subject: [PATCH 06/10] implement central solution for restoring Toolchain class 'constants' --- easybuild/toolchains/compiler/cuda.py | 2 +- .../toolchains/compiler/inteliccifort.py | 10 ++-- easybuild/toolchains/linalg/acml.py | 13 ++--- easybuild/toolchains/linalg/intelmkl.py | 14 ++---- easybuild/tools/toolchain/toolchain.py | 35 ++++++++++++- test/framework/toolchain.py | 49 ++++++++++++------- 6 files changed, 82 insertions(+), 41 deletions(-) diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 9397a9a584..ffbe45099c 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -96,7 +96,7 @@ def _set_compiler_flags(self): self.variables.nappend('CUDA_CXXFLAGS', cuda_flags) # add gencode compiler flags to list of flags for compiler variables - for gencode_val in self.options['cuda_gencode']: + for gencode_val in self.options.get('cuda_gencode', []): gencode_option = 'gencode %s' % gencode_val self.variables.nappend('CUDA_CFLAGS', gencode_option) self.variables.nappend('CUDA_CXXFLAGS', gencode_option) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 7532468998..d024b0aaad 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -88,8 +88,11 @@ class IntelIccIfort(Compiler): } LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related - # keep track of original value, needs to be restored every time since we append to class 'constant' LIB_MULTITHREAD - _INIT_LIB_MULTITHREAD = LIB_MULTITHREAD[:] + + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + self.CLASS_CONSTANTS_TO_RESTORE.append('LIB_MULTITHREAD') + super(IntelIccIfort, self).__init__(*args, **kwargs) def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" @@ -106,9 +109,6 @@ def _set_compiler_vars(self): raise EasyBuildError("_set_compiler_vars: mismatch between icc version %s and ifort version %s", icc_version, ifort_version) - # reset LIB_MULTITHREAD every time, - # to avoid problems when multiple Intel compilers versions are used in a single session - self.LIB_MULTITHREAD = self._INIT_LIB_MULTITHREAD[:] if LooseVersion(icc_version) < LooseVersion('2011'): self.LIB_MULTITHREAD.insert(1, "guide") diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 13876777f3..26ced341b4 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -45,9 +45,6 @@ class Acml(LinAlg): BLAS_MODULE_NAME = ['ACML'] BLAS_LIB = ['acml'] BLAS_LIB_MT = ['acml_mp'] - # keep track of original values, need to be restored every time since we append to class 'constants' BLAS_LIB* - _INIT_BLAS_LIB = BLAS_LIB[:] - _INIT_BLAS_LIB_MT = BLAS_LIB_MT[:] # is completed in _set_blas_variables, depends on compiler used BLAS_LIB_DIR = [] @@ -60,6 +57,11 @@ class Acml(LinAlg): TC_CONSTANT_GCC: ['gfortran64', 'gfortran64_mp'], } + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + self.CLASS_CONSTANTS_TO_RESTORE.extend(['BLAS_LIB', 'BLAS_LIB_MT']) + super(Acml, self).__init__(*args, **kwargs) + def _set_blas_variables(self): """Fix the map a bit""" if self.options.get('32bit', None): @@ -75,11 +77,6 @@ def _set_blas_variables(self): raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " "with compiler family %s", self.COMPILER_FAMILY) - # reset BLAS_LIB(_MT) every time, to avoid problems when multiple ACML versions are used in a single session - # full list of libraries is highly dependent on ACML version and toolchain compiler (ifort, gfortran, ...) - self.BLAS_LIB = self._INIT_BLAS_LIB[:] - self.BLAS_LIB_MT = self._INIT_BLAS_LIB_MT[:] - # version before 5.x still featured the acml_mv library ver = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(ver) < LooseVersion("5"): diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index ae3a3e7914..0e1eceefaf 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -73,10 +73,11 @@ class IntelMKL(LinAlg): SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} - # keep track of original values, need to be restored every time since we append to class 'constants' SCALAPACK_LIB* - _INIT_SCALAPACK_LIB = SCALAPACK_LIB[:] - _INIT_SCALAPACK_LIB_MT = SCALAPACK_LIB_MT[:] - _INIT_SCALAPACK_LIB_MAP = copy.deepcopy(SCALAPACK_LIB_MAP) + + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + self.CLASS_CONSTANTS_TO_RESTORE.extend(['SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) + super(IntelMKL, self).__init__(*args, **kwargs) def _set_blas_variables(self): """Fix the map a bit""" @@ -154,11 +155,6 @@ def _set_blacs_variables(self): super(IntelMKL, self)._set_blacs_variables() def _set_scalapack_variables(self): - # reset SCALAPACK_LIB* every time, to avoid problems when multiple imkl versions are used in a single session - self.SCALAPACK_LIB = self._INIT_SCALAPACK_LIB[:] - self.SCALAPACK_LIB_MT = self._INIT_SCALAPACK_LIB_MT[:] - self.SCALAPACK_LIB_MAP = copy.deepcopy(self._INIT_SCALAPACK_LIB_MAP) - imkl_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] if LooseVersion(imkl_version) < LooseVersion('10.3'): self.SCALAPACK_LIB.append("mkl_solver%(lp64)s_sequential") diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a5cad572be..b63e3265d3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -30,7 +30,7 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +import copy import os from vsc.utils import fancylogger @@ -58,6 +58,10 @@ class Toolchain(object): VERSION = None TOOLCHAIN_FAMILY = None + # list of class 'constants' that should be restored for every new instance of this class + CLASS_CONSTANTS_TO_RESTORE = [] + CLASS_CONSTANT_COPIES = {} + # class method def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" @@ -95,6 +99,9 @@ def __init__(self, name=None, version=None, mns=None): self.vars = None + self._copy_class_constants() + self._restore_class_constants() + self.modules_tool = modules_tool() self.mns = mns self.mod_full_name = None @@ -122,6 +129,32 @@ def base_init(self): if hasattr(self, 'LINKER_TOGGLE_STATIC_DYNAMIC'): self.variables.LINKER_TOGGLE_STATIC_DYNAMIC = self.LINKER_TOGGLE_STATIC_DYNAMIC + def _copy_class_constants(self): + """Copy class constants that needs to be restored again when a new instance is created.""" + # this only needs to be done the first time (for this class, taking inheritance into account is key) + key = self.__class__ + if key not in self.CLASS_CONSTANT_COPIES: + self.CLASS_CONSTANT_COPIES[key] = {} + for cst in self.CLASS_CONSTANTS_TO_RESTORE: + if hasattr(self, cst): + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) + else: + raise EasyBuildError("Class constant '%s' to be restored does not exist", cst) + + self.log.debug("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) + + def _restore_class_constants(self): + """Restored class constants that need to be restored when a new instance is created.""" + key = self.__class__ + for cst in self.CLASS_CONSTANT_COPIES[key]: + newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) + if hasattr(self, cst): + self.log.debug("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) + else: + self.log.debug("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + + setattr(self, cst, newval) + def get_variable(self, name, typ=str): """Get value for specified variable. typ: indicates what type of return value is expected""" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 7fee7d0831..faf64fface 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -622,50 +622,65 @@ def test_old_new_iccifort(self): tmpdir2, imkl_module_path2, imkl_module_txt2 = self.setup_sandbox_for_intel_fftw(imklver='10.2.6.038') # incl. -lguide - libblas_mt_old = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" - libblas_mt_old += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" + libblas_mt_ictce3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide - libblas_mt_new = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" - libblas_mt_new += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + libblas_mt_ictce4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce4 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" # incl. -lmkl_solver* - libscalack_old = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64" - libscalack_old += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + libscalack_ictce3 = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64" + libscalack_ictce3 += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" # no -lmkl_solver* - libscalack_new = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + libscalack_ictce4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + libblas_mt_goolfc = "-lopenblas -lgfortran" + libscalack_goolfc = "-lscalapack -lopenblas -lgfortran" + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) + modules_tool().purge() tc = self.get_toolchain('ictce', version='4.1.13') tc.prepare() - self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) - self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='3.2.2.u3') tc.prepare() - self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) - self.assertTrue(libscalack_old in os.environ['LIBSCALAPACK']) + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='4.1.13') tc.prepare() - self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_new) - self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) modules_tool().purge() tc = self.get_toolchain('ictce', version='3.2.2.u3') tc.prepare() - self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_old) - self.assertTrue(libscalack_old in os.environ['LIBSCALAPACK']) + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) modules_tool().purge() - libscalack_new = libscalack_new.replace('_lp64', '_ilp64') + libscalack_ictce4 = libscalack_ictce4.replace('_lp64', '_ilp64') tc = self.get_toolchain('ictce', version='4.1.13') opts = {'i8': True} tc.set_options(opts) tc.prepare() - self.assertTrue(libscalack_new in os.environ['LIBSCALAPACK']) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) # cleanup shutil.rmtree(tmpdir1) From 0671a434e3cb9b9310991c4a1b213f56f9be3ee2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 16:56:44 +0200 Subject: [PATCH 07/10] cleanup --- easybuild/toolchains/linalg/acml.py | 1 + easybuild/toolchains/linalg/intelmkl.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 26ced341b4..0dc899b2e6 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -43,6 +43,7 @@ class Acml(LinAlg): Provides ACML BLAS/LAPACK support. """ BLAS_MODULE_NAME = ['ACML'] + # full list of libraries is highly dependent on ACML version and toolchain compiler (ifort, gfortran, ...) BLAS_LIB = ['acml'] BLAS_LIB_MT = ['acml_mp'] diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 0e1eceefaf..78cc87de2f 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -67,15 +67,16 @@ class IntelMKL(LinAlg): BLACS_LIB_STATIC = True SCALAPACK_MODULE_NAME = ['imkl'] - SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] - SCALAPACK_LIB_GROUP = True - SCALAPACK_LIB_STATIC = True SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} + SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] + SCALAPACK_LIB_GROUP = True + SCALAPACK_LIB_STATIC = True def __init__(self, *args, **kwargs): """Toolchain constructor.""" + self.CLASS_CONSTANTS_TO_RESTORE.append('BLAS_LIB_MAP') self.CLASS_CONSTANTS_TO_RESTORE.extend(['SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) super(IntelMKL, self).__init__(*args, **kwargs) @@ -106,10 +107,10 @@ def _set_blas_variables(self): if self.options.get('32bit', None): # 32bit - self.BLAS_LIB_MAP.update({"lp64":''}) # FIXME + self.BLAS_LIB_MAP.update({"lp64":''}) if self.options.get('i8', None): # ilp64/i8 - self.BLAS_LIB_MAP.update({"lp64":'_ilp64'}) # FIXME + self.BLAS_LIB_MAP.update({"lp64":'_ilp64'}) # CPP / CFLAGS self.variables.nappend_el('CFLAGS', 'DMKL_ILP64') From d4c5fc6673715eafdf152ee3ce595d2652fdc969 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 17:54:57 +0200 Subject: [PATCH 08/10] don't define Toolchain class constant as empty list, not inheritance-safe --- easybuild/toolchains/compiler/inteliccifort.py | 2 +- easybuild/toolchains/linalg/acml.py | 2 +- easybuild/toolchains/linalg/intelmkl.py | 3 +-- easybuild/tools/toolchain/toolchain.py | 12 ++++++++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index d024b0aaad..d2a71dac7b 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -91,7 +91,7 @@ class IntelIccIfort(Compiler): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.CLASS_CONSTANTS_TO_RESTORE.append('LIB_MULTITHREAD') + self.add_class_constants_to_restore(['LIB_MULTITHREAD']) super(IntelIccIfort, self).__init__(*args, **kwargs) def _set_compiler_vars(self): diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 0dc899b2e6..e6f8da07e5 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -60,7 +60,7 @@ class Acml(LinAlg): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.CLASS_CONSTANTS_TO_RESTORE.extend(['BLAS_LIB', 'BLAS_LIB_MT']) + self.add_class_constants_to_restore(['BLAS_LIB', 'BLAS_LIB_MT']) super(Acml, self).__init__(*args, **kwargs) def _set_blas_variables(self): diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 78cc87de2f..c10969ed21 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -76,8 +76,7 @@ class IntelMKL(LinAlg): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.CLASS_CONSTANTS_TO_RESTORE.append('BLAS_LIB_MAP') - self.CLASS_CONSTANTS_TO_RESTORE.extend(['SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) + self.add_class_constants_to_restore(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) super(IntelMKL, self).__init__(*args, **kwargs) def _set_blas_variables(self): diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index b63e3265d3..9548ceb41a 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -59,7 +59,7 @@ class Toolchain(object): TOOLCHAIN_FAMILY = None # list of class 'constants' that should be restored for every new instance of this class - CLASS_CONSTANTS_TO_RESTORE = [] + CLASS_CONSTANTS_TO_RESTORE = None CLASS_CONSTANT_COPIES = {} # class method @@ -99,6 +99,7 @@ def __init__(self, name=None, version=None, mns=None): self.vars = None + self.add_class_constants_to_restore([]) # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised self._copy_class_constants() self._restore_class_constants() @@ -115,6 +116,13 @@ def __init__(self, name=None, version=None, mns=None): self.mod_short_name = self.mns.det_short_module_name(tc_dict) self.init_modpaths = self.mns.det_init_modulepaths(tc_dict) + def add_class_constants_to_restore(self, names): + """Add given constants to list of class constants to restore with each new instance.""" + if self.CLASS_CONSTANTS_TO_RESTORE is None: + self.CLASS_CONSTANTS_TO_RESTORE = names[:] + else: + self.CLASS_CONSTANTS_TO_RESTORE.extend(names) + def base_init(self): if not hasattr(self, 'log'): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -139,7 +147,7 @@ def _copy_class_constants(self): if hasattr(self, cst): self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) else: - raise EasyBuildError("Class constant '%s' to be restored does not exist", cst) + raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) self.log.debug("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) From 77cf7f6c6718a2d8e9399fa46b9d828e9c7f0947 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jun 2015 18:50:13 +0200 Subject: [PATCH 09/10] drop unused import --- easybuild/toolchains/linalg/intelmkl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index c10969ed21..2102c6e02e 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -28,7 +28,6 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ -import copy from distutils.version import LooseVersion from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP From cf16ee292980ac42a6f73e86ff234f6ce594a9df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 8 Jun 2015 10:01:35 +0200 Subject: [PATCH 10/10] fix remarks --- .../toolchains/compiler/inteliccifort.py | 3 ++- easybuild/toolchains/linalg/acml.py | 3 ++- easybuild/toolchains/linalg/intelmkl.py | 3 ++- easybuild/tools/toolchain/toolchain.py | 25 +++++++++++-------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index d2a71dac7b..e9af5dc722 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -91,7 +91,8 @@ class IntelIccIfort(Compiler): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.add_class_constants_to_restore(['LIB_MULTITHREAD']) + class_constants = kwargs.setdefault('class_constants', []) + class_constants.append('LIB_MULTITHREAD') super(IntelIccIfort, self).__init__(*args, **kwargs) def _set_compiler_vars(self): diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index e6f8da07e5..fb7c5def68 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -60,7 +60,8 @@ class Acml(LinAlg): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.add_class_constants_to_restore(['BLAS_LIB', 'BLAS_LIB_MT']) + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB', 'BLAS_LIB_MT']) super(Acml, self).__init__(*args, **kwargs) def _set_blas_variables(self): diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 2102c6e02e..81ae047c63 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -75,7 +75,8 @@ class IntelMKL(LinAlg): def __init__(self, *args, **kwargs): """Toolchain constructor.""" - self.add_class_constants_to_restore(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) super(IntelMKL, self).__init__(*args, **kwargs) def _set_blas_variables(self): diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 9548ceb41a..907bb65734 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -77,7 +77,7 @@ def _is_toolchain_for(cls, name): _is_toolchain_for = classmethod(_is_toolchain_for) - def __init__(self, name=None, version=None, mns=None): + def __init__(self, name=None, version=None, mns=None, class_constants=None): """Toolchain constructor.""" self.base_init() @@ -99,9 +99,7 @@ def __init__(self, name=None, version=None, mns=None): self.vars = None - self.add_class_constants_to_restore([]) # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised - self._copy_class_constants() - self._restore_class_constants() + self._init_class_constants(class_constants) self.modules_tool = modules_tool() self.mns = mns @@ -116,14 +114,8 @@ def __init__(self, name=None, version=None, mns=None): self.mod_short_name = self.mns.det_short_module_name(tc_dict) self.init_modpaths = self.mns.det_init_modulepaths(tc_dict) - def add_class_constants_to_restore(self, names): - """Add given constants to list of class constants to restore with each new instance.""" - if self.CLASS_CONSTANTS_TO_RESTORE is None: - self.CLASS_CONSTANTS_TO_RESTORE = names[:] - else: - self.CLASS_CONSTANTS_TO_RESTORE.extend(names) - def base_init(self): + """Initialise missing class attributes (log, options, variables).""" if not hasattr(self, 'log'): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -137,6 +129,17 @@ def base_init(self): if hasattr(self, 'LINKER_TOGGLE_STATIC_DYNAMIC'): self.variables.LINKER_TOGGLE_STATIC_DYNAMIC = self.LINKER_TOGGLE_STATIC_DYNAMIC + def _init_class_constants(self, class_constants): + """Initialise class 'constants'.""" + # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised + if class_constants is None: + self.CLASS_CONSTANTS_TO_RESTORE = [] + else: + self.CLASS_CONSTANTS_TO_RESTORE = class_constants[:] + + self._copy_class_constants() + self._restore_class_constants() + def _copy_class_constants(self): """Copy class constants that needs to be restored again when a new instance is created.""" # this only needs to be done the first time (for this class, taking inheritance into account is key)