Optional TOML configs support (spotify#2457)

See the added docs for usage.

doc/configuration.rst

@@ -1,18 +1,35 @@
 Configuration
 =============
 
-All configuration can be done by adding configuration files. They are looked for in:
+All configuration can be done by adding configuration files.
 
- * ``/etc/luigi/client.cfg``
- * ``luigi.cfg`` (or its legacy name ``client.cfg``) in your current working directory
- * ``LUIGI_CONFIG_PATH`` environment variable
+Supported config parsers:
+* ``cfg`` (default)
+* ``toml``
 
-in increasing order of preference. The order only matters in case of key conflicts (see docs for ConfigParser.read_). These files are meant for both the client and ``luigid``. If you decide to specify your own configuration you should make sure that both the client and ``luigid`` load it properly.
+You can choose right parser via ``LUIGI_CONFIG_PARSER`` environment variable. For example, ``LUIGI_CONFIG_PARSER=toml``.
+
+Default (cfg) parser are looked for in:
+
+* ``/etc/luigi/client.cfg`` (deprecated)
+* ``/etc/luigi/luigi.cfg``
+* ``client.cfg`` (deprecated)
+* ``luigi.cfg``
+* ``LUIGI_CONFIG_PATH`` environment variable
+
+`TOML <https://github.com/toml-lang/toml>`_ parser are looked for in:
+
+* ``/etc/luigi/luigi.toml``
+* ``luigi.toml``
+* ``LUIGI_CONFIG_PATH`` environment variable
+
+Both config lists increase in priority (from low to high). The order only matters in case of key conflicts (see docs for ConfigParser.read_). These files are meant for both the client and ``luigid``. If you decide to specify your own configuration you should make sure that both the client and ``luigid`` load it properly.
 
 .. _ConfigParser.read: https://docs.python.org/3.6/library/configparser.html#configparser.ConfigParser.read
 
-The config file is broken into sections, each controlling a different part of the config. Example configuration file:
+The config file is broken into sections, each controlling a different part of the config.
 
+Example cfg config:
 
 .. code:: ini
 
@@ -23,6 +40,17 @@ The config file is broken into sections, each controlling a different part of th
     [core]
     scheduler_host=luigi-host.mycompany.foo
 
+Example toml config:
+
+.. code:: python
+
+    [hadoop]
+    version = "cdh4"
+    streaming-jar = "/usr/lib/hadoop-xyz/hadoop-streaming-xyz-123.jar"
+
+    [core]
+    scheduler_host = "luigi-host.mycompany.foo"
+
 
 .. _ParamConfigIngestion:
 

luigi/configuration/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from .cfg_parser import LuigiConfigParser
+from .core import get_config, add_config_path
+from .toml_parser import LuigiTomlParser
+
+
+__all__ = [
+    'add_config_path',
+    'get_config',
+    'LuigiConfigParser',
+    'LuigiTomlParser',
+]

luigi/configuration/base_parser.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+
+
+# IMPORTANT: don't inherit from `object`!
+# ConfigParser have some troubles in this case.
+# More info: https://stackoverflow.com/a/19323238
+class BaseParser:
+    @classmethod
+    def instance(cls, *args, **kwargs):
+        """ Singleton getter """
+        if cls._instance is None:
+            cls._instance = cls(*args, **kwargs)
+            loaded = cls._instance.reload()
+            logging.getLogger('luigi-interface').info('Loaded %r', loaded)
+
+        return cls._instance
+
+    @classmethod
+    def add_config_path(cls, path):
+        cls._config_paths.append(path)
+        cls.reload()
+
+    @classmethod
+    def reload(cls):
+        return cls.instance().read(cls._config_paths)

luigi/configuration.py → luigi/configuration/cfg_parser.py

@@ -29,7 +29,6 @@
 See :doc:`/configuration` for more info.
 """
 
-import logging
 import os
 import warnings
 
@@ -38,37 +37,19 @@
 except ImportError:
     from configparser import ConfigParser, NoOptionError, NoSectionError
 
+from .base_parser import BaseParser
 
-class LuigiConfigParser(ConfigParser):
+
+class LuigiConfigParser(BaseParser, ConfigParser):
     NO_DEFAULT = object()
+    enabled = True
     _instance = None
     _config_paths = [
         '/etc/luigi/client.cfg',  # Deprecated old-style global luigi config
         '/etc/luigi/luigi.cfg',
         'client.cfg',  # Deprecated old-style local luigi config
         'luigi.cfg',
     ]
-    if 'LUIGI_CONFIG_PATH' in os.environ:
-        config_file = os.environ['LUIGI_CONFIG_PATH']
-        if not os.path.isfile(config_file):
-            warnings.warn("LUIGI_CONFIG_PATH points to a file which does not exist. Invalid file: {path}".format(path=config_file))
-        else:
-            _config_paths.append(config_file)
-
-    @classmethod
-    def add_config_path(cls, path):
-        cls._config_paths.append(path)
-        cls.reload()
-
-    @classmethod
-    def instance(cls, *args, **kwargs):
-        """ Singleton getter """
-        if cls._instance is None:
-            cls._instance = cls(*args, **kwargs)
-            loaded = cls._instance.reload()
-            logging.getLogger('luigi-interface').info('Loaded %r', loaded)
-
-        return cls._instance
 
     @classmethod
     def reload(cls):
@@ -124,10 +105,3 @@ def set(self, section, option, value=None):
             ConfigParser.add_section(self, section)
 
         return ConfigParser.set(self, section, option, value)
-
-
-def get_config():
-    """
-    Convenience method (for backwards compatibility) for accessing config singleton.
-    """
-    return LuigiConfigParser.instance()

luigi/configuration/core.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012-2015 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+import os
+import warnings
+
+from .cfg_parser import LuigiConfigParser
+from .toml_parser import LuigiTomlParser
+
+
+logger = logging.getLogger('luigi-interface')
+
+
+PARSERS = {
+    'cfg': LuigiConfigParser,
+    'conf': LuigiConfigParser,
+    'ini': LuigiConfigParser,
+    'toml': LuigiTomlParser,
+}
+
+# select parser via env var
+DEFAULT_PARSER = 'cfg'
+PARSER = os.environ.get('LUIGI_CONFIG_PARSER', DEFAULT_PARSER)
+if PARSER not in PARSERS:
+    warnings.warn("Invalid parser: {parser}".format(parser=PARSER))
+    PARSER = DEFAULT_PARSER
+
+
+def get_config(parser=PARSER):
+    """Get configs singleton for parser
+    """
+
+    parser_class = PARSERS[parser]
+    if not parser_class.enabled:
+        logger.error((
+                "Parser not installed yet. "
+                "Please, install luigi with required parser:\n"
+                "pip install luigi[{parser}]"
+            ).format(parser)
+        )
+
+    return parser_class.instance()
+
+
+def add_config_path(path):
+    """Select config parser by file extension and add path into parser.
+    """
+    if not os.path.isfile(path):
+        warnings.warn("Config file does not exist: {path}".format(path=path))
+        return False
+
+    # select parser by file extension
+    _base, ext = os.path.splitext(path)
+    if ext and ext[1:] in PARSERS:
+        parser_class = PARSERS[ext[1:]]
+    else:
+        parser_class = PARSERS[PARSER]
+
+    # add config path to parser
+    parser_class.add_config_path(path)
+    return True
+
+
+if 'LUIGI_CONFIG_PATH' in os.environ:
+    add_config_path(os.environ['LUIGI_CONFIG_PATH'])

luigi/configuration/toml_parser.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018 Cindicator Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os.path
+
+try:
+    import toml
+except ImportError:
+    toml = False
+
+from .base_parser import BaseParser
+
+
+class LuigiTomlParser(BaseParser):
+    NO_DEFAULT = object()
+    enabled = bool(toml)
+    data = dict()
+    _instance = None
+    _config_paths = [
+        '/etc/luigi/luigi.toml',
+        'luigi.toml',
+    ]
+
+    @staticmethod
+    def _update_data(data, new_data):
+        if not new_data:
+            return data
+        if not data:
+            return new_data
+        for section, content in new_data.items():
+            if section not in data:
+                data[section] = dict()
+            data[section].update(content)
+        return data
+
+    def read(self, config_paths):
+        self.data = dict()
+        for path in config_paths:
+            if os.path.isfile(path):
+                self.data = self._update_data(self.data, toml.load(path))
+        return self.data
+
+    def get(self, section, option, default=NO_DEFAULT, **kwargs):
+        try:
+            return self.data[section][option]
+        except KeyError:
+            if default is self.NO_DEFAULT:
+                raise
+            return default
+
+    def getboolean(self, section, option, default=NO_DEFAULT):
+        return self.get(section, option, default)
+
+    def getint(self, section, option, default=NO_DEFAULT):
+        return self.get(section, option, default)
+
+    def getfloat(self, section, option, default=NO_DEFAULT):
+        return self.get(section, option, default)
+
+    def getintdict(self, section):
+        return self.data.get(section, {})
+
+    def set(self, section, option, value=None):
+        if section not in self.data:
+            self.data[section] = {}
+        self.data[section][option] = value
+
+    def __getitem__(self, name):
+        return self.data[name]

luigi/contrib/s3.py

@@ -486,7 +486,7 @@ def _get_s3_config(self, key=None):
         defaults = dict(configuration.get_config().defaults())
         try:
             config = dict(configuration.get_config().items('s3'))
-        except NoSectionError:
+        except (NoSectionError, KeyError):
             return {}
         # So what ports etc can be read without us having to specify all dtypes
         for k, v in six.iteritems(config):

luigi/parameter.py

@@ -168,7 +168,7 @@ def _get_value_from_config(self, section, name):
 
         try:
             value = conf.get(section, name)
-        except (NoSectionError, NoOptionError):
+        except (NoSectionError, NoOptionError, KeyError):
             return _no_value
 
         return self.parse(value)

setup.py

@@ -58,6 +58,7 @@ def get_static_files(path):
     license='Apache License 2.0',
     packages=[
         'luigi',
+        'luigi.configuration',
         'luigi.contrib',
         'luigi.contrib.hdfs',
         'luigi.tools'
@@ -75,6 +76,9 @@ def get_static_files(path):
         ]
     },
     install_requires=install_requires,
+    extras_require={
+        'toml': ['toml<2.0.0'],
+    },
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',