diff --git a/aiida/backends/profile.py b/aiida/backends/profile.py index 5783d2e22a..d42205534d 100644 --- a/aiida/backends/profile.py +++ b/aiida/backends/profile.py @@ -25,6 +25,7 @@ def load_profile(profile=None): Load the profile. This function is called by load_dbenv and SHOULD NOT be called by the user by hand. """ + from aiida.common.log import configure_logging from aiida.manage import get_config if settings.LOAD_PROFILE_CALLED: @@ -43,6 +44,9 @@ def load_profile(profile=None): settings.AIIDADB_PROFILE = profile + # Reconfigure the logging to make sure that profile specific logging configuration options are taken into account + configure_logging() + profile = config.get_profile(profile) # Check if AIIDADB_BACKEND is set and if not error (with message) diff --git a/aiida/backends/tests/manage/configuration/test_options.py b/aiida/backends/tests/manage/configuration/test_options.py index 7035b23992..cfb699319e 100644 --- a/aiida/backends/tests/manage/configuration/test_options.py +++ b/aiida/backends/tests/manage/configuration/test_options.py @@ -13,7 +13,9 @@ from __future__ import absolute_import from aiida.backends.testbase import AiidaTestCase +from aiida.backends.tests.utils.configuration import with_temporary_config_instance from aiida.manage.configuration.options import get_option, get_option_names, parse_option, Option, CONFIG_OPTIONS +from aiida.manage.configuration import get_config, get_config_option class TestConfigurationOptions(AiidaTestCase): @@ -52,3 +54,40 @@ def test_options(self): self.assertEqual(option.valid_values, option_settings['valid_values']) self.assertEqual(option.default, option_settings['default']) self.assertEqual(option.description, option_settings['description']) + + @with_temporary_config_instance + def test_get_config_option_default(self): + """Tests that `get_option` return option default if not specified globally or for current profile.""" + option_name = 'logging.aiida_loglevel' + option = get_option(option_name) + + # If we haven't set the option explicitly, `get_config_option` should return the option default + option_value = get_config_option(option_name) + self.assertEqual(option_value, option.default) + + @with_temporary_config_instance + def test_get_config_option_profile_specific(self): + """Tests that `get_option` correctly gets a configuration option if specified for the current profile.""" + config = get_config() + profile = config.current_profile + + option_name = 'logging.aiida_loglevel' + option_value_profile = 'WARNING' + + # Setting a specific value for the current profile which should then be returned by `get_config_option` + config.option_set(option_name, option_value_profile, scope=profile.name) + option_value = get_config_option(option_name) + self.assertEqual(option_value, option_value_profile) + + @with_temporary_config_instance + def test_get_config_option_global(self): + """Tests that `get_option` correctly agglomerates upwards and so retrieves globally set config options.""" + config = get_config() + + option_name = 'logging.aiida_loglevel' + option_value_global = 'CRITICAL' + + # Setting a specific value globally which should then be returned by `get_config_option` due to agglomeration + config.option_set(option_name, option_value_global) + option_value = get_config_option(option_name) + self.assertEqual(option_value, option_value_global) diff --git a/aiida/common/log.py b/aiida/common/log.py index f079450646..81db8fd249 100644 --- a/aiida/common/log.py +++ b/aiida/common/log.py @@ -12,8 +12,9 @@ from __future__ import print_function from __future__ import absolute_import -import copy +import collections import logging +import types from aiida.manage import get_config_option @@ -115,44 +116,44 @@ def emit(self, record): 'filters': ['testing'] }, 'dblogger': { - 'level': get_config_option('logging.db_loglevel'), + 'level': lambda: get_config_option('logging.db_loglevel'), 'class': 'aiida.common.log.DBLogHandler', }, }, 'loggers': { 'aiida': { 'handlers': ['console', 'dblogger'], - 'level': get_config_option('logging.aiida_loglevel'), + 'level': lambda: get_config_option('logging.aiida_loglevel'), 'propagate': False, }, 'tornado': { 'handlers': ['console'], - 'level': get_config_option('logging.tornado_loglevel'), + 'level': lambda: get_config_option('logging.tornado_loglevel'), 'propagate': False, }, 'plumpy': { 'handlers': ['console'], - 'level': get_config_option('logging.plumpy_loglevel'), + 'level': lambda: get_config_option('logging.plumpy_loglevel'), 'propagate': False, }, 'kiwipy': { 'handlers': ['console'], - 'level': get_config_option('logging.kiwipy_loglevel'), + 'level': lambda: get_config_option('logging.kiwipy_loglevel'), 'propagate': False, }, 'paramiko': { 'handlers': ['console'], - 'level': get_config_option('logging.paramiko_loglevel'), + 'level': lambda: get_config_option('logging.paramiko_loglevel'), 'propagate': False, }, 'alembic': { 'handlers': ['console'], - 'level': get_config_option('logging.alembic_loglevel'), + 'level': lambda: get_config_option('logging.alembic_loglevel'), 'propagate': False, }, 'sqlalchemy': { 'handlers': ['console'], - 'level': get_config_option('logging.sqlalchemy_loglevel'), + 'level': lambda: get_config_option('logging.sqlalchemy_loglevel'), 'propagate': False, 'qualname': 'sqlalchemy.engine', }, @@ -163,6 +164,26 @@ def emit(self, record): } +def evaluate_logging_configuration(dictionary): + """Recursively evaluate the logging configuration, calling lambdas when encountered. + + This allows the configuration options that are dependent on the active profile to be loaded lazily. + + :return: evaluated logging configuration dictionary + """ + result = {} + + for key, value in dictionary.items(): + if isinstance(value, collections.Mapping): + result[key] = evaluate_logging_configuration(value) + elif isinstance(value, types.LambdaType): + result[key] = value() + else: + result[key] = value + + return result + + def configure_logging(daemon=False, daemon_log_file=None): """ Setup the logging by retrieving the LOGGING dictionary from aiida and passing it to @@ -177,7 +198,7 @@ def configure_logging(daemon=False, daemon_log_file=None): """ from logging.config import dictConfig - config = copy.deepcopy(LOGGING) + config = evaluate_logging_configuration(LOGGING) daemon_handler_name = 'daemon_log_file' # Add the daemon file handler to all loggers if daemon=True diff --git a/aiida/manage/configuration/__init__.py b/aiida/manage/configuration/__init__.py index 5a3dcb90d3..40353c8f88 100644 --- a/aiida/manage/configuration/__init__.py +++ b/aiida/manage/configuration/__init__.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- # pylint: disable=undefined-variable,wildcard-import,global-statement """Modules related to the configuration of an AiiDA instance.""" +from __future__ import absolute_import from .config import * +from .options import * from .profile import * from .utils import * CONFIG = None -__all__ = (config.__all__ + profile.__all__ + utils.__all__ + ('get_config',)) +__all__ = (config.__all__ + options.__all__ + profile.__all__ + utils.__all__ + ('get_config', 'get_config_option')) def get_config(): @@ -29,3 +31,36 @@ def get_config(): CONFIG = load_config() return CONFIG + + +def get_config_option(option_name): + """Return the value for the given configuration option. + + This function will attempt to load the value of the option as defined for the current profile or otherwise as + defined configuration wide. If no configuration is yet loaded, this function will fall back on the default that may + be defined for the option itself. This is useful for options that need to be defined at loading time of AiiDA when + no configuration is yet loaded or may not even yet exist. In cases where one expects a profile to be loaded, + preference should be given to retrieving the option through the Config instance and its `option_get` method. + + :param option_name: the name of the configuration option + :return: option value as specified for the profile/configuration if loaded, otherwise option default + """ + from aiida.common import exceptions + + option = options.get_option(option_name) + + try: + config = get_config() + except exceptions.ConfigurationError: + value = option.default + else: + if config.current_profile: + # Try to get the option for the profile, but do not return the option default + value_profile = config.option_get(option_name, scope=config.current_profile.name, default=False) + else: + value_profile = None + + # Value is the profile value if defined or otherwise the global value, which will be None if not set + value = value_profile if value_profile else config.option_get(option_name) + + return value diff --git a/aiida/manage/configuration/config.py b/aiida/manage/configuration/config.py index bfe740c053..e1aa7f0175 100644 --- a/aiida/manage/configuration/config.py +++ b/aiida/manage/configuration/config.py @@ -6,7 +6,6 @@ import os import shutil -from aiida.common import exceptions from aiida.common import json from .migrations import CURRENT_CONFIG_VERSION, OLDEST_COMPATIBLE_CONFIG_VERSION @@ -146,6 +145,8 @@ def validate_profile(self, name): :param name: name of the profile: :raises ProfileConfigurationError: if the name is not found in the configuration file """ + from aiida.common import exceptions + if name not in self.profile_names: raise exceptions.ProfileConfigurationError('profile `{}` does not exist'.format(name)) diff --git a/aiida/manage/configuration/utils.py b/aiida/manage/configuration/utils.py index d76e4a1080..472922c4b0 100644 --- a/aiida/manage/configuration/utils.py +++ b/aiida/manage/configuration/utils.py @@ -5,42 +5,11 @@ import io import os -from aiida.common import exceptions -from aiida.common import json - from .config import Config from .migrations import check_and_migrate_config -from .options import get_option from .settings import DEFAULT_CONFIG_FILE_NAME -__all__ = ('get_config_option', 'load_config') - - -def get_config_option(option_name): - """Return the value for the given configuration option. - - This function will attempt to load the value of the option as defined for the current profile or otherwise as - defined configuration wide. If no configuration is yet loaded, this function will fall back on the default that may - be defined for the option itself. This is useful for options that need to be defined at loading time of AiiDA when - no configuration is yet loaded or may not even yet exist. In cases where one expects a profile to be loaded, - preference should be given to retrieving the option through the Config instance and its `option_get` method. - - :param option_name: the name of the configuration option - :return: option value as specified for the profile/configuration if loaded, otherwise option default - """ - option = get_option(option_name) - - try: - config = load_config() - except exceptions.ConfigurationError: - value = option.default - else: - if config.current_profile: - value = config.option_get(option_name, scope=config.current_profile.name) - else: - value = config.option_get(option_name) - - return value +__all__ = ('load_config',) def load_config(create=False): @@ -53,6 +22,8 @@ def load_config(create=False): """ from .settings import AIIDA_CONFIG_FOLDER from aiida.backends.settings import IN_RT_DOC_MODE, DUMMY_CONF_FILE + from aiida.common import exceptions + from aiida.common import json if IN_RT_DOC_MODE: return DUMMY_CONF_FILE