Merge pull request #3555 from Flamefire/perf_improvement

Performance improvements for easyconfig parsing and eb startup

easybuild/base/generaloption.py

@@ -40,13 +40,30 @@
 from functools import reduce
 from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values
 from optparse import SUPPRESS_HELP as nohelp  # supported in optparse of python v2.4
-from optparse import gettext as _gettext  # this is gettext.gettext normally
 
 from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
 from easybuild.base.optcomplete import autocomplete, CompleterOption
 from easybuild.tools.py2vs3 import StringIO, configparser, string_type
 from easybuild.tools.utilities import mk_rst_table, nub, shell_quote
 
+try:
+    import gettext
+    eb_translation = None
+
+    def get_translation():
+        global eb_translation
+        if not eb_translation:
+            # Finding a translation is expensive, so do only once
+            domain = gettext.textdomain()
+            eb_translation = gettext.translation(domain, gettext.bindtextdomain(domain), fallback=True)
+        return eb_translation
+
+    def _gettext(message):
+        return get_translation().gettext(message)
+except ImportError:
+    def _gettext(message):
+        return message
+
 
 HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config']
 

easybuild/framework/easyconfig/templates.py

@@ -31,7 +31,6 @@
 :author: Fotis Georgatos (Uni.Lu, NTUA)
 :author: Kenneth Hoste (Ghent University)
 """
-import copy
 import re
 import platform
 
@@ -242,10 +241,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
             raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
 
     # step 2: define *ver and *shortver templates
-    for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
+    if TEMPLATE_SOFTWARE_VERSIONS:
 
-        # copy to avoid changing original list below
-        deps = copy.copy(config.get('dependencies', []))
+        name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS)
+        deps = config.get('dependencies', [])
 
         # also consider build dependencies for *ver and *shortver templates;
         # we need to be a bit careful here, because for iterative installations
@@ -262,15 +261,22 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
             # only consider build dependencies when we're actually in iterative mode!
             if 'builddependencies' in config.iterate_options:
                 if config.iterating:
-                    deps.extend(config.get('builddependencies', []))
+                    build_deps = config.get('builddependencies')
+                else:
+                    build_deps = None
             else:
-                deps.extend(config.get('builddependencies', []))
-
+                build_deps = config.get('builddependencies')
+            if build_deps:
+                # Don't use += to avoid changing original list
+                deps = deps + build_deps
             # include all toolchain deps (e.g. CUDAcore component in fosscuda);
             # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
-            if config._toolchain is not None:
-                if config._toolchain.tcdeps is not None:
+            if config._toolchain is not None and config._toolchain.tcdeps:
+                # If we didn't create a new list above do it here
+                if build_deps:
                     deps.extend(config._toolchain.tcdeps)
+                else:
+                    deps = deps + config._toolchain.tcdeps
 
         for dep in deps:
             if isinstance(dep, dict):
@@ -292,15 +298,16 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
             else:
                 raise EasyBuildError("Unexpected type for dependency: %s", dep)
 
-            if isinstance(dep_name, string_type) and dep_name.lower() == name.lower() and dep_version:
-                dep_version = pick_dep_version(dep_version)
-                template_values['%sver' % pref] = dep_version
-                dep_version_parts = dep_version.split('.')
-                template_values['%smajver' % pref] = dep_version_parts[0]
-                if len(dep_version_parts) > 1:
-                    template_values['%sminver' % pref] = dep_version_parts[1]
-                template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
-                break
+            if isinstance(dep_name, string_type) and dep_version:
+                pref = name_to_prefix.get(dep_name.lower())
+                if pref:
+                    dep_version = pick_dep_version(dep_version)
+                    template_values['%sver' % pref] = dep_version
+                    dep_version_parts = dep_version.split('.')
+                    template_values['%smajver' % pref] = dep_version_parts[0]
+                    if len(dep_version_parts) > 1:
+                        template_values['%sminver' % pref] = dep_version_parts[1]
+                    template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
 
     # step 3: add remaining from config
     for name in TEMPLATE_NAMES_CONFIG:

easybuild/toolchains/linalg/libsci.py

@@ -59,18 +59,19 @@ class LibSci(LinAlg):
     BLACS_MODULE_NAME = []
     SCALAPACK_MODULE_NAME = []
 
-    def _get_software_root(self, name):
+    def _get_software_root(self, name, required=True):
         """Get install prefix for specified software name; special treatment for Cray modules."""
         if name == 'cray-libsci':
             # Cray-provided LibSci module
             env_var = 'CRAY_LIBSCI_PREFIX_DIR'
             root = os.getenv(env_var, None)
             if root is None:
-                raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var)
+                if required:
+                    raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var)
             else:
                 self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
         else:
-            root = super(LibSci, self)._get_software_root(name)
+            root = super(LibSci, self)._get_software_root(name, required=required)
 
         return root
 

easybuild/tools/run.py

@@ -281,6 +281,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False
 
     # read remaining data (all of it)
     output = get_output_from_process(proc)
+    proc.stdout.close()
     if cmd_log:
         cmd_log.write(output)
         cmd_log.close()

easybuild/tools/toolchain/mpi.py

@@ -293,10 +293,9 @@ def mpi_cmd_for(self, cmd, nr_ranks):
             # for Intel MPI, try to determine impi version
             # this fails when it's done too early (before modules for toolchain/dependencies are loaded),
             # but it's safe to ignore this
-            try:
-                mpi_version = self.get_software_version(self.MPI_MODULE_NAME)[0]
-            except EasyBuildError as err:
-                self.log.debug("Ignoring error when trying to determine %s version: %s", self.MPI_MODULE_NAME, err)
+            mpi_version = self.get_software_version(self.MPI_MODULE_NAME, required=False)[0]
+            if not mpi_version:
+                self.log.debug("Ignoring error when trying to determine %s version", self.MPI_MODULE_NAME)
                 # impi version is required to determine correct MPI command template,
                 # so we have to return early if we couldn't determine the impi version...
                 return None

easybuild/tools/toolchain/toolchain.py

@@ -403,33 +403,35 @@ def get_software_root(self, names):
         """Try to get the software root for all names"""
         return self._get_software_multiple(names, self._get_software_root)
 
-    def get_software_version(self, names):
+    def get_software_version(self, names, required=True):
         """Try to get the software version for all names"""
-        return self._get_software_multiple(names, self._get_software_version)
+        return self._get_software_multiple(names, self._get_software_version, required=required)
 
-    def _get_software_multiple(self, names, function):
+    def _get_software_multiple(self, names, function, required=True):
         """Execute function of each of names"""
         if isinstance(names, (str,)):
             names = [names]
         res = []
         for name in names:
-            res.append(function(name))
+            res.append(function(name, required=required))
         return res
 
-    def _get_software_root(self, name):
+    def _get_software_root(self, name, required=True):
         """Try to get the software root for name"""
         root = get_software_root(name)
         if root is None:
-            raise EasyBuildError("get_software_root software root for %s was not found in environment", name)
+            if required:
+                raise EasyBuildError("get_software_root software root for %s was not found in environment", name)
         else:
             self.log.debug("get_software_root software root %s for %s was found in environment", root, name)
         return root
 
-    def _get_software_version(self, name):
+    def _get_software_version(self, name, required=True):
         """Try to get the software version for name"""
         version = get_software_version(name)
         if version is None:
-            raise EasyBuildError("get_software_version software version for %s was not found in environment", name)
+            if required:
+                raise EasyBuildError("get_software_version software version for %s was not found in environment", name)
         else:
             self.log.debug("get_software_version software version %s for %s was found in environment", version, name)
 

test/framework/easyblock.py

@@ -1677,6 +1677,9 @@ def test_extensions_sanity_check(self):
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
         toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
 
+        # Do this before loading the easyblock to check the non-translated output below
+        os.environ['LC_ALL'] = 'C'
+
         # this import only works here, since EB_toy is a test easyblock
         from easybuild.easyblocks.toy import EB_toy
 

test/framework/easyconfig.py

@@ -3081,26 +3081,29 @@ def test_template_constant_dict(self):
             "    'arch=fooarch': '1.8.0-foo',",
             "  })",
             "]",
+            "builddependencies = [",
+            "  ('CMake', '3.18.4'),",
+            "]",
         ])
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         write_file(test_ec, toy_ec_txt)
 
-        # only perform shallow/quick parse (as is done in list_software function)
-        ec = EasyConfigParser(filename=test_ec).get_config_dict()
-
         expected = {
+            'bitbucket_account': 'toy',
+            'github_account': 'toy',
             'javamajver': '1',
             'javaminver': '8',
             'javashortver': '1.8',
             'javaver': '1.8.0_221',
-            'module_name': None,
+            'module_name': 'toy/0.01-deps',
             'name': 'toy',
             'namelower': 'toy',
             'nameletter': 't',
             'toolchain_name': 'system',
             'toolchain_version': 'system',
             'nameletterlower': 't',
+            'parallel': None,
             'pymajver': '3',
             'pyminver': '7',
             'pyshortver': '3.7',
@@ -3109,9 +3112,38 @@ def test_template_constant_dict(self):
             'version_major': '0',
             'version_major_minor': '0.01',
             'version_minor': '01',
+            'versionprefix': '',
             'versionsuffix': '-deps',
         }
+
+        # proper EasyConfig instance
+        ec = EasyConfig(test_ec)
+
+        # CMake should *not* be included, since it's a build-only dependency
+        dep_names = [x['name'] for x in ec['dependencies']]
+        self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names)
+        res = template_constant_dict(ec)
+        dep_names = [x['name'] for x in ec['dependencies']]
+        self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names)
+
+        self.assertTrue('arch' in res)
+        arch = res.pop('arch')
+        self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern))
+
+        self.assertEqual(res, expected)
+
+        # only perform shallow/quick parse (as is done in list_software function)
+        ec = EasyConfigParser(filename=test_ec).get_config_dict()
+
+        expected['module_name'] = None
+        for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'):
+            del expected[key]
+
+        dep_names = [x[0] for x in ec['dependencies']]
+        self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names)
         res = template_constant_dict(ec)
+        dep_names = [x[0] for x in ec['dependencies']]
+        self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names)
 
         self.assertTrue('arch' in res)
         arch = res.pop('arch')

test/framework/modules.py

@@ -1036,6 +1036,7 @@ def test_modules_tool_stateless(self):
             load_err_msg = "Unable to locate a modulefile"
 
         # GCC/4.6.3 is *not* an available Core module
+        os.environ['LC_ALL'] = 'C'
         self.assertErrorRegex(EasyBuildError, load_err_msg, self.modtool.load, ['GCC/4.6.3'])
 
         # GCC/6.4.0-2.28 is one of the available Core modules

test/framework/options.py

@@ -3409,12 +3409,10 @@ def mk_eb_test_cmd(self, args):
         """Construct test command for 'eb' with given options."""
 
         # make sure that location to 'easybuild.main' is included in $PYTHONPATH
+        pythonpath = os.getenv('PYTHONPATH')
+        pythonpath = [pythonpath] if pythonpath else []
         easybuild_loc = os.path.dirname(os.path.dirname(easybuild.main.__file__))
-        try:
-            pythonpath = easybuild_loc + ':' + os.environ['PYTHONPATH']
-        except KeyError:
-            pythonpath = easybuild_loc
-        os.environ['PYTHONPATH'] = pythonpath
+        os.environ['PYTHONPATH'] = ':'.join([easybuild_loc] + pythonpath)
 
         return '; '.join([
             "cd %s" % self.test_prefix,

test/framework/toolchain.py

@@ -1422,6 +1422,26 @@ def test_prepare_deps_external(self):
         self.assertEqual(modules.get_software_root('foobar'), '/foo/bar')
         self.assertEqual(modules.get_software_version('toy'), '1.2.3')
 
+    def test_get_software_version(self):
+        """Test that get_software_version works"""
+        os.environ['EBROOTTOY'] = '/foo/bar'
+        os.environ['EBVERSIONTOY'] = '1.2.3'
+        os.environ['EBROOTFOOBAR'] = '/foo/bar'
+        os.environ['EBVERSIONFOOBAR'] = '4.5'
+        tc = self.get_toolchain('GCC', version='6.4.0-2.28')
+        self.assertEqual(tc.get_software_version('toy'), ['1.2.3'])
+        self.assertEqual(tc.get_software_version(['toy']), ['1.2.3'])
+        self.assertEqual(tc.get_software_version(['toy', 'foobar']), ['1.2.3', '4.5'])
+        # Non existing modules raise an error
+        self.assertErrorRegex(EasyBuildError, 'non-existing was not found',
+                              tc.get_software_version, 'non-existing')
+        self.assertErrorRegex(EasyBuildError, 'non-existing was not found',
+                              tc.get_software_version, ['toy', 'non-existing', 'foobar'])
+        # Can use required=False to avoid
+        self.assertEqual(tc.get_software_version('non-existing', required=False), [None])
+        self.assertEqual(tc.get_software_version(['toy', 'non-existing', 'foobar'], required=False),
+                         ['1.2.3', None, '4.5'])
+
     def test_old_new_iccifort(self):
         """Test whether preparing for old/new Intel compilers works correctly."""
         self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2018.1.163')