Ensure that configuration options are considered for logging config

The configuration knows various options to change the logging
configuration, however, these were not respected for two reasons:

 * Logging configuration was not lazily evaluated
 * Globally configured options were not agglomerated

The first problem was caused by the fact that the logging configuration
is evaluated upon loading the `aiida` module, at which point the profile
is not necessarily loaded yet, causing the `get_config_option` functions
to return the option defaults. The solution is to have the dictionary
lazily evaluated by using lambdas, which are resolved when
`configure_logging` is called. Finally, we make sure this function is
called each time the profile is set.

The second problem arose from the fact that if a profile is defined, the
`get_config_option` only returned the config value if explicitly set for
that profile and otherwise it would return the option default. This
means that if the option was defined globally for the configuration it
was ignored. This is now corrected where if the current profile does not
explicitly define a value for the option but it is globally defined, the
global value is returned.

aiida/backends/profile.py

@@ -25,6 +25,7 @@ def load_profile(profile=None):
     Load the profile. This function is called by load_dbenv and SHOULD NOT
     be called by the user by hand.
     """
+    from aiida.common.log import configure_logging
     from aiida.manage import get_config
 
     if settings.LOAD_PROFILE_CALLED:
@@ -43,6 +44,9 @@ def load_profile(profile=None):
 
         settings.AIIDADB_PROFILE = profile
 
+    # Reconfigure the logging to make sure that profile specific logging configuration options are taken into account
+    configure_logging()
+
     profile = config.get_profile(profile)
 
     # Check if AIIDADB_BACKEND is set and if not error (with message)

aiida/backends/tests/manage/configuration/test_options.py

@@ -13,7 +13,9 @@
 from __future__ import absolute_import
 
 from aiida.backends.testbase import AiidaTestCase
+from aiida.backends.tests.utils.configuration import with_temporary_config_instance
 from aiida.manage.configuration.options import get_option, get_option_names, parse_option, Option, CONFIG_OPTIONS
+from aiida.manage.configuration import get_config, get_config_option
 
 
 class TestConfigurationOptions(AiidaTestCase):
@@ -52,3 +54,40 @@ def test_options(self):
             self.assertEqual(option.valid_values, option_settings['valid_values'])
             self.assertEqual(option.default, option_settings['default'])
             self.assertEqual(option.description, option_settings['description'])
+
+    @with_temporary_config_instance
+    def test_get_config_option_default(self):
+        """Tests that `get_option` return option default if not specified globally or for current profile."""
+        option_name = 'logging.aiida_loglevel'
+        option = get_option(option_name)
+
+        # If we haven't set the option explicitly, `get_config_option` should return the option default
+        option_value = get_config_option(option_name)
+        self.assertEqual(option_value, option.default)
+
+    @with_temporary_config_instance
+    def test_get_config_option_profile_specific(self):
+        """Tests that `get_option` correctly gets a configuration option if specified for the current profile."""
+        config = get_config()
+        profile = config.current_profile
+
+        option_name = 'logging.aiida_loglevel'
+        option_value_profile = 'WARNING'
+
+        # Setting a specific value for the current profile which should then be returned by `get_config_option`
+        config.option_set(option_name, option_value_profile, scope=profile.name)
+        option_value = get_config_option(option_name)
+        self.assertEqual(option_value, option_value_profile)
+
+    @with_temporary_config_instance
+    def test_get_config_option_global(self):
+        """Tests that `get_option` correctly agglomerates upwards and so retrieves globally set config options."""
+        config = get_config()
+
+        option_name = 'logging.aiida_loglevel'
+        option_value_global = 'CRITICAL'
+
+        # Setting a specific value globally which should then be returned by `get_config_option` due to agglomeration
+        config.option_set(option_name, option_value_global)
+        option_value = get_config_option(option_name)
+        self.assertEqual(option_value, option_value_global)

aiida/common/log.py

@@ -12,8 +12,9 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
-import copy
+import collections
 import logging
+import types
 
 from aiida.manage import get_config_option
 
@@ -115,44 +116,44 @@ def emit(self, record):
             'filters': ['testing']
         },
         'dblogger': {
-            'level': get_config_option('logging.db_loglevel'),
+            'level': lambda: get_config_option('logging.db_loglevel'),
             'class': 'aiida.common.log.DBLogHandler',
         },
     },
     'loggers': {
         'aiida': {
             'handlers': ['console', 'dblogger'],
-            'level': get_config_option('logging.aiida_loglevel'),
+            'level': lambda: get_config_option('logging.aiida_loglevel'),
             'propagate': False,
         },
         'tornado': {
             'handlers': ['console'],
-            'level': get_config_option('logging.tornado_loglevel'),
+            'level': lambda: get_config_option('logging.tornado_loglevel'),
             'propagate': False,
         },
         'plumpy': {
             'handlers': ['console'],
-            'level': get_config_option('logging.plumpy_loglevel'),
+            'level': lambda: get_config_option('logging.plumpy_loglevel'),
             'propagate': False,
         },
         'kiwipy': {
             'handlers': ['console'],
-            'level': get_config_option('logging.kiwipy_loglevel'),
+            'level': lambda: get_config_option('logging.kiwipy_loglevel'),
             'propagate': False,
         },
         'paramiko': {
             'handlers': ['console'],
-            'level': get_config_option('logging.paramiko_loglevel'),
+            'level': lambda: get_config_option('logging.paramiko_loglevel'),
             'propagate': False,
         },
         'alembic': {
             'handlers': ['console'],
-            'level': get_config_option('logging.alembic_loglevel'),
+            'level': lambda: get_config_option('logging.alembic_loglevel'),
             'propagate': False,
         },
         'sqlalchemy': {
             'handlers': ['console'],
-            'level': get_config_option('logging.sqlalchemy_loglevel'),
+            'level': lambda: get_config_option('logging.sqlalchemy_loglevel'),
             'propagate': False,
             'qualname': 'sqlalchemy.engine',
         },
@@ -163,6 +164,26 @@ def emit(self, record):
 }
 
 
+def evaluate_logging_configuration(dictionary):
+    """Recursively evaluate the logging configuration, calling lambdas when encountered.
+
+    This allows the configuration options that are dependent on the active profile to be loaded lazily.
+
+    :return: evaluated logging configuration dictionary
+    """
+    result = {}
+
+    for key, value in dictionary.items():
+        if isinstance(value, collections.Mapping):
+            result[key] = evaluate_logging_configuration(value)
+        elif isinstance(value, types.LambdaType):
+            result[key] = value()
+        else:
+            result[key] = value
+
+    return result
+
+
 def configure_logging(daemon=False, daemon_log_file=None):
     """
     Setup the logging by retrieving the LOGGING dictionary from aiida and passing it to
@@ -177,7 +198,7 @@ def configure_logging(daemon=False, daemon_log_file=None):
     """
     from logging.config import dictConfig
 
-    config = copy.deepcopy(LOGGING)
+    config = evaluate_logging_configuration(LOGGING)
     daemon_handler_name = 'daemon_log_file'
 
     # Add the daemon file handler to all loggers if daemon=True

aiida/manage/configuration/__init__.py

@@ -1,14 +1,16 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=undefined-variable,wildcard-import,global-statement
 """Modules related to the configuration of an AiiDA instance."""
+from __future__ import absolute_import
 
 from .config import *
+from .options import *
 from .profile import *
 from .utils import *
 
 CONFIG = None
 
-__all__ = (config.__all__ + profile.__all__ + utils.__all__ + ('get_config',))
+__all__ = (config.__all__ + options.__all__ + profile.__all__ + utils.__all__ + ('get_config', 'get_config_option'))
 
 
 def get_config():
@@ -29,3 +31,36 @@ def get_config():
         CONFIG = load_config()
 
     return CONFIG
+
+
+def get_config_option(option_name):
+    """Return the value for the given configuration option.
+
+    This function will attempt to load the value of the option as defined for the current profile or otherwise as
+    defined configuration wide. If no configuration is yet loaded, this function will fall back on the default that may
+    be defined for the option itself. This is useful for options that need to be defined at loading time of AiiDA when
+    no configuration is yet loaded or may not even yet exist. In cases where one expects a profile to be loaded,
+    preference should be given to retrieving the option through the Config instance and its `option_get` method.
+
+    :param option_name: the name of the configuration option
+    :return: option value as specified for the profile/configuration if loaded, otherwise option default
+    """
+    from aiida.common import exceptions
+
+    option = options.get_option(option_name)
+
+    try:
+        config = get_config()
+    except exceptions.ConfigurationError:
+        value = option.default
+    else:
+        if config.current_profile:
+            # Try to get the option for the profile, but do not return the option default
+            value_profile = config.option_get(option_name, scope=config.current_profile.name, default=False)
+        else:
+            value_profile = None
+
+        # Value is the profile value if defined or otherwise the global value, which will be None if not set
+        value = value_profile if value_profile else config.option_get(option_name)
+
+    return value

aiida/manage/configuration/config.py

@@ -6,7 +6,6 @@
 import os
 import shutil
 
-from aiida.common import exceptions
 from aiida.common import json
 
 from .migrations import CURRENT_CONFIG_VERSION, OLDEST_COMPATIBLE_CONFIG_VERSION
@@ -146,6 +145,8 @@ def validate_profile(self, name):
         :param name: name of the profile:
         :raises ProfileConfigurationError: if the name is not found in the configuration file
         """
+        from aiida.common import exceptions
+
         if name not in self.profile_names:
             raise exceptions.ProfileConfigurationError('profile `{}` does not exist'.format(name))
 

aiida/manage/configuration/utils.py

@@ -5,42 +5,11 @@
 import io
 import os
 
-from aiida.common import exceptions
-from aiida.common import json
-
 from .config import Config
 from .migrations import check_and_migrate_config
-from .options import get_option
 from .settings import DEFAULT_CONFIG_FILE_NAME
 
-__all__ = ('get_config_option', 'load_config')
-
-
-def get_config_option(option_name):
-    """Return the value for the given configuration option.
-
-    This function will attempt to load the value of the option as defined for the current profile or otherwise as
-    defined configuration wide. If no configuration is yet loaded, this function will fall back on the default that may
-    be defined for the option itself. This is useful for options that need to be defined at loading time of AiiDA when
-    no configuration is yet loaded or may not even yet exist. In cases where one expects a profile to be loaded,
-    preference should be given to retrieving the option through the Config instance and its `option_get` method.
-
-    :param option_name: the name of the configuration option
-    :return: option value as specified for the profile/configuration if loaded, otherwise option default
-    """
-    option = get_option(option_name)
-
-    try:
-        config = load_config()
-    except exceptions.ConfigurationError:
-        value = option.default
-    else:
-        if config.current_profile:
-            value = config.option_get(option_name, scope=config.current_profile.name)
-        else:
-            value = config.option_get(option_name)
-
-    return value
+__all__ = ('load_config',)
 
 
 def load_config(create=False):
@@ -53,6 +22,8 @@ def load_config(create=False):
     """
     from .settings import AIIDA_CONFIG_FOLDER
     from aiida.backends.settings import IN_RT_DOC_MODE, DUMMY_CONF_FILE
+    from aiida.common import exceptions
+    from aiida.common import json
 
     if IN_RT_DOC_MODE:
         return DUMMY_CONF_FILE