Merge pull request #3 from boegel/checksums-itemdict

sync branch with develop (& sync conflicts)

easybuild/framework/easyconfig/types.py

@@ -301,6 +301,27 @@ def to_name_version_dict(spec):
     _log.nosupport("to_name_version_dict; use to_toolchain_dict instead.", '3.0')
 
 
+def to_list_of_strings(value):
+    """
+    Convert specified value to a list of strings, if possible.
+
+    Supported: single string value, tuple of string values.
+    """
+    res = None
+
+    # if value is already of correct type, we don't need to change anything
+    if isinstance(value, list) and all(isinstance(s, basestring) for s in value):
+        res = value
+    elif isinstance(value, basestring):
+        res = [value]
+    elif isinstance(value, tuple) and all(isinstance(s, basestring) for s in value):
+        res = list(value)
+    else:
+        raise EasyBuildError("Don't know how to convert provided value to a list of strings: %s", value)
+
+    return res
+
+
 def to_list_of_strings_and_tuples(spec):
     """
     Convert a 'list of lists and strings' to a 'list of tuples and strings'
@@ -491,6 +512,7 @@ def ensure_iterable_license_specs(specs):
 DEPENDENCIES = (list, as_hashable({'elem_types': [DEPENDENCY_DICT]}))
 
 TUPLE_OF_STRINGS = (tuple, as_hashable({'elem_types': [str]}))
+LIST_OF_STRINGS = (list, as_hashable({'elem_types': [str]}))
 STRING_OR_TUPLE_LIST = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS]}))
 SANITY_CHECK_PATHS_DICT = (dict, as_hashable({
     'elem_types': {
@@ -509,15 +531,16 @@ def ensure_iterable_license_specs(specs):
 ))
 CHECKSUMS = (list, as_hashable({'elem_types': [CHECKSUM, DICT_CHECKSUM]}))
 
-CHECKABLE_TYPES = [CHECKSUM, DICT_CHECKSUM, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, TOOLCHAIN_DICT,
-                   SANITY_CHECK_PATHS_DICT, STRING_OR_TUPLE_LIST, TUPLE_OF_STRINGS]
+CHECKABLE_TYPES = [CHECKSUM, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, DICT_CHECKSUM, LIST_OF_STRINGS,
+                   SANITY_CHECK_PATHS_DICT, STRING_OR_TUPLE_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
 
 # easy types, that can be verified with isinstance
 EASY_TYPES = [basestring, bool, dict, int, list, str, tuple]
 
 # type checking is skipped for easyconfig parameters names not listed in PARAMETER_TYPES
 PARAMETER_TYPES = {
     'checksums': CHECKSUMS,
+    'docurls': LIST_OF_STRINGS,
     'name': basestring,
     'osdependencies': STRING_OR_TUPLE_LIST,
     'patches': STRING_OR_TUPLE_LIST,
@@ -536,6 +559,7 @@ def ensure_iterable_license_specs(specs):
     str: str,
     CHECKSUMS: to_checksums,
     DEPENDENCIES: to_dependencies,
+    LIST_OF_STRINGS: to_list_of_strings,
     TOOLCHAIN_DICT: to_toolchain_dict,
     SANITY_CHECK_PATHS_DICT: to_sanity_check_paths_dict,
     STRING_OR_TUPLE_LIST: to_list_of_strings_and_tuples,

easybuild/toolchains/foss.py

@@ -27,6 +27,7 @@
 
 :author: Kenneth Hoste (Ghent University)
 """
+from distutils.version import LooseVersion
 
 from easybuild.toolchains.gompi import Gompi
 from easybuild.toolchains.golf import Golf
@@ -39,3 +40,16 @@ class Foss(Gompi, OpenBLAS, ScaLAPACK, Fftw):
     """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW."""
     NAME = 'foss'
     SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME]
+
+    def is_deprecated(self):
+        """Return whether or not this toolchain is deprecated."""
+        # foss toolchains older than foss/2016a are deprecated
+        # take into account that foss/2016.x is always < foss/2016a according to LooseVersion;
+        # foss/2016.01 & co are not deprecated yet...
+        foss_ver = LooseVersion(self.version)
+        if foss_ver < LooseVersion('2016a') and foss_ver < LooseVersion('2016.01'):
+            deprecated = True
+        else:
+            deprecated = False
+
+        return deprecated

easybuild/toolchains/gompi.py

@@ -40,8 +40,14 @@ class Gompi(GccToolchain, OpenMPI):
 
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
+        gompi_ver = LooseVersion(self.version)
         # deprecate oldest gompi toolchains (versions 1.x)
-        if LooseVersion(self.version) < LooseVersion('2000'):
+        if gompi_ver < LooseVersion('2000'):
+            deprecated = True
+        # gompi toolchains older than gompi/2016a are deprecated
+        # take into account that gompi/2016.x is always < gompi/2016a according to LooseVersion;
+        # gompi/2016.01 & co are not deprecated yet...
+        elif gompi_ver < LooseVersion('2016a') and gompi_ver < LooseVersion('2016.01'):
             deprecated = True
         else:
             deprecated = False

test/framework/options.py

@@ -1053,6 +1053,9 @@ def test_from_pr(self):
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
         os.close(fd)
 
+        # need to allow triggering deprecated behaviour because of old toolchain (< gompi/2016a)
+        self.allow_deprecated_behaviour()
+
         tmpdir = tempfile.mkdtemp()
         args = [
             # PR for foss/2015a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/1239/files
@@ -1161,6 +1164,9 @@ def test_from_pr_x(self):
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
         os.close(fd)
 
+        # need to allow triggering deprecated behaviour because of old toolchain (< gompi/2016a)
+        self.allow_deprecated_behaviour()
+
         args = [
             # PR for foss/2015a, see https://github.com/easybuilders/easybuild-easyconfigs/pull/1239/files
             '--from-pr=1239',
@@ -1172,9 +1178,11 @@ def test_from_pr_x(self):
         ]
         try:
             self.mock_stdout(True)
+            self.mock_stderr(True)
             self.eb_main(args, do_build=True, raise_error=True, testing=False)
             stdout = self.get_stdout()
             self.mock_stdout(False)
+            self.mock_stderr(False)
 
             msg_regexs = [
                 re.compile(r"^== Build succeeded for 1 out of 1", re.M),

test/framework/robot.py

@@ -692,6 +692,9 @@ def test_det_easyconfig_paths_from_pr(self):
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
         os.close(fd)
 
+        # need to allow triggering deprecated behaviour because of old toolchain (< gompi/2016a)
+        self.allow_deprecated_behaviour()
+
         test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
 
         test_ec = 'toy-0.0-deps.eb'

test/framework/type_checking.py

@@ -34,9 +34,10 @@
 from easybuild.framework.easyconfig.types import as_hashable, check_element_types, check_key_types, check_known_keys
 from easybuild.framework.easyconfig.types import check_required_keys, check_type_of_param_value, convert_value_type
 from easybuild.framework.easyconfig.types import DEPENDENCIES, DEPENDENCY_DICT, ensure_iterable_license_specs
-from easybuild.framework.easyconfig.types import TOOLCHAIN_DICT, SANITY_CHECK_PATHS_DICT, STRING_OR_TUPLE_LIST
+from easybuild.framework.easyconfig.types import LIST_OF_STRINGS, SANITY_CHECK_PATHS_DICT, STRING_OR_TUPLE_LIST
+from easybuild.framework.easyconfig.types import TOOLCHAIN_DICT
 from easybuild.framework.easyconfig.types import is_value_of_type, to_checksums, to_dependencies, to_dependency
-from easybuild.framework.easyconfig.types import to_list_of_strings_and_tuples, to_toolchain_dict
+from easybuild.framework.easyconfig.types import to_list_of_strings, to_list_of_strings_and_tuples, to_toolchain_dict
 from easybuild.framework.easyconfig.types import to_sanity_check_paths_dict
 from easybuild.tools.build_log import EasyBuildError
 
@@ -187,11 +188,18 @@ def test_convert_value_type(self):
         # 1.6 can't be parsed as an int (yields "invalid literal for int() with base 10" error)
         self.assertErrorRegex(EasyBuildError, "Converting type of .* failed", convert_value_type, '1.6', int)
 
+        # to list of strings
+        self.assertEqual(convert_value_type('foo', LIST_OF_STRINGS), ['foo'])
+        self.assertEqual(convert_value_type(('foo', 'bar'), LIST_OF_STRINGS), ['foo', 'bar'])
+        self.assertEqual(convert_value_type((), LIST_OF_STRINGS), [])
+
         # idempotency
         self.assertEqual(convert_value_type('foo', basestring), 'foo')
         self.assertEqual(convert_value_type('foo', str), 'foo')
         self.assertEqual(convert_value_type(100, int), 100)
         self.assertEqual(convert_value_type(1.6, float), 1.6)
+        self.assertEqual(convert_value_type(['foo', 'bar'], LIST_OF_STRINGS), ['foo', 'bar'])
+        self.assertEqual(convert_value_type([], LIST_OF_STRINGS), [])
 
         # complex types
         dep = [{'GCC': '1.2.3', 'versionsuffix': 'foo'}]
@@ -343,6 +351,14 @@ def test_is_value_of_type(self):
         self.assertFalse(is_value_of_type(1, str))
         self.assertFalse(is_value_of_type("foo", int))
 
+        # list of strings check
+        self.assertTrue(is_value_of_type([], LIST_OF_STRINGS))
+        self.assertTrue(is_value_of_type(['foo', 'bar'], LIST_OF_STRINGS))
+        self.assertTrue(is_value_of_type([''], LIST_OF_STRINGS))
+        self.assertFalse(is_value_of_type(123, LIST_OF_STRINGS))
+        self.assertFalse(is_value_of_type('foo', LIST_OF_STRINGS))
+        self.assertFalse(is_value_of_type(('foo', 'bar'), LIST_OF_STRINGS))
+
         # toolchain type check
         self.assertTrue(is_value_of_type({'name': 'intel', 'version': '2015a'}, TOOLCHAIN_DICT))
         # version value should be string, not int
@@ -503,6 +519,28 @@ def test_check_element_types(self):
         # errors
         self.assertErrorRegex(EasyBuildError, "Don't know how to check element types .*", check_element_types, 1, [])
 
+    def test_to_list_of_strings(self):
+        """Test to_list_of_strings function."""
+        # no conversion if value type is already correct
+        self.assertEqual(to_list_of_strings([]), [])
+        self.assertEqual(to_list_of_strings(['foo']), ['foo'])
+        self.assertEqual(to_list_of_strings(['foo', 'bar', 'baz']), ['foo', 'bar', 'baz'])
+
+        # single string is converted to a single-element list
+        self.assertEqual(to_list_of_strings('foo'), ['foo'])
+        self.assertEqual(to_list_of_strings(''), [''])
+
+        # tuple of strings is converted to list of strings
+        self.assertEqual(to_list_of_strings(['foo', 'bar']), ['foo', 'bar'])
+        self.assertEqual(to_list_of_strings(['foo']), ['foo'])
+        self.assertEqual(to_list_of_strings(()), [])
+
+        # proper error reporting for other values
+        error_pattern = r"Don't know how to convert provided value to a list of strings: "
+        self.assertErrorRegex(EasyBuildError, error_pattern + '123', to_list_of_strings, 123)
+        self.assertErrorRegex(EasyBuildError, error_pattern + 'True', to_list_of_strings, True)
+        self.assertErrorRegex(EasyBuildError, error_pattern, to_list_of_strings, [('foo', 'bar')])
+
     def test_to_list_of_strings_and_tuples(self):
         """Test to_list_of_strings_and_tuples function."""
         # no conversion, already right type