♻️ REFACTOR: Profile storage backend configuration

aiida/backends/djsite/settings.py

@@ -23,21 +23,19 @@
 if PROFILE is None:
     raise exceptions.ProfileConfigurationError('no profile has been loaded')
 
-if PROFILE.database_backend != 'django':
+if PROFILE.storage_backend != 'django':
     raise exceptions.ProfileConfigurationError(
-        f'incommensurate database backend `{PROFILE.database_backend}` for profile `{PROFILE.name}`'
+        f'incommensurate database backend `{PROFILE.storage_backend}` for profile `{PROFILE.name}`'
     )
 
-PROFILE_CONF = PROFILE.dictionary
-
 DATABASES = {
     'default': {
-        'ENGINE': f'django.db.backends.{PROFILE.database_engine}',
-        'NAME': PROFILE.database_name,
-        'PORT': PROFILE.database_port,
-        'HOST': PROFILE.database_hostname,
-        'USER': PROFILE.database_username,
-        'PASSWORD': PROFILE.database_password,
+        'ENGINE': f"django.db.backends.{PROFILE.storage_config['database_engine']}",
+        'NAME': PROFILE.storage_config['database_name'],
+        'PORT': PROFILE.storage_config['database_port'],
+        'HOST': PROFILE.storage_config['database_hostname'],
+        'USER': PROFILE.storage_config['database_username'],
+        'PASSWORD': PROFILE.storage_config['database_password'],
     }
 }
 

aiida/backends/testbase.py

@@ -47,10 +47,10 @@ def get_backend_class(cls):
 
         # Freeze the __impl_class after the first run
         if not hasattr(cls, '__impl_class'):
-            if PROFILE.database_backend == BACKEND_SQLA:
+            if PROFILE.storage_backend == BACKEND_SQLA:
                 from aiida.backends.sqlalchemy.testbase import SqlAlchemyTests
                 cls.__impl_class = SqlAlchemyTests
-            elif PROFILE.database_backend == BACKEND_DJANGO:
+            elif PROFILE.storage_backend == BACKEND_DJANGO:
                 from aiida.backends.djsite.db.testbase import DjangoTests
                 cls.__impl_class = DjangoTests
             else:
@@ -248,7 +248,7 @@ def get_default_user(**kwargs):
     :returns: the :py:class:`~aiida.orm.User`
     """
     from aiida.manage.configuration import get_config
-    email = get_config().current_profile.default_user
+    email = get_config().current_profile.default_user_email
 
     if kwargs.pop('email', None):
         raise ValueError('Do not specify the user email (must coincide with default user email of profile).')

aiida/backends/utils.py

@@ -8,10 +8,15 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Backend-agnostic utility functions"""
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from aiida.manage.configuration.profile import Profile
+
 AIIDA_ATTRIBUTE_SEP = '.'
 
 
-def create_sqlalchemy_engine(profile, **kwargs):
+def create_sqlalchemy_engine(profile: 'Profile', **kwargs):
     """Create SQLAlchemy engine (to be used for QueryBuilder queries)
 
     :param kwargs: keyword arguments that will be passed on to `sqlalchemy.create_engine`.
@@ -24,16 +29,16 @@ def create_sqlalchemy_engine(profile, **kwargs):
 
     # The hostname may be `None`, which is a valid value in the case of peer authentication for example. In this case
     # it should be converted to an empty string, because otherwise the `None` will be converted to string literal "None"
-    hostname = profile.database_hostname or ''
-    separator = ':' if profile.database_port else ''
+    hostname = profile.storage_config['database_hostname'] or ''
+    separator = ':' if profile.storage_config['database_port'] else ''
 
     engine_url = 'postgresql://{user}:{password}@{hostname}{separator}{port}/{name}'.format(
         separator=separator,
-        user=profile.database_username,
-        password=profile.database_password,
+        user=profile.storage_config['database_username'],
+        password=profile.storage_config['database_password'],
         hostname=hostname,
-        port=profile.database_port,
-        name=profile.database_name
+        port=profile.storage_config['database_port'],
+        name=profile.storage_config['database_name']
     )
     return create_engine(
         engine_url, json_serializer=json.dumps, json_deserializer=json.loads, future=True, encoding='utf-8', **kwargs

aiida/cmdline/commands/cmd_setup.py

@@ -51,20 +51,23 @@ def setup(
     from aiida import orm
     from aiida.manage.configuration import get_config
 
-    profile.database_engine = db_engine
-    profile.database_backend = db_backend
-    profile.database_name = db_name
-    profile.database_port = db_port
-    profile.database_hostname = db_host
-    profile.database_username = db_username
-    profile.database_password = db_password
+    profile.set_storage(
+        db_backend, {
+            'database_engine': db_engine,
+            'database_hostname': db_host,
+            'database_port': db_port,
+            'database_name': db_name,
+            'database_username': db_username,
+            'database_password': db_password,
+            'repository_uri': f'file://{repository}',
+        }
+    )
     profile.broker_protocol = broker_protocol
     profile.broker_username = broker_username
     profile.broker_password = broker_password
     profile.broker_host = broker_host
     profile.broker_port = broker_port
     profile.broker_virtual_host = broker_virtual_host
-    profile.repository_uri = f'file://{repository}'
 
     config = get_config()
 
@@ -115,7 +118,7 @@ def setup(
     )
     if created:
         user.store()
-    profile.default_user = user.email
+    profile.default_user_email = user.email
     config.update_profile(profile)
     config.store()
 

aiida/cmdline/commands/cmd_status.py

@@ -101,7 +101,12 @@ def verdi_status(print_traceback, no_rmq):
     dbgen = backend_manager.get_schema_generation_database()
     dbver = backend_manager.get_schema_version_backend()
     database_data = [
-        profile.database_name, dbgen, dbver, profile.database_username, profile.database_hostname, profile.database_port
+        profile.storage_config['database_name'],
+        dbgen,
+        dbver,
+        profile.storage_config['database_username'],
+        profile.storage_config['database_hostname'],
+        profile.storage_config['database_port'],
     ]
     try:
         with override_log_level():  # temporarily suppress noisy logging

aiida/cmdline/commands/cmd_user.py

@@ -26,7 +26,7 @@ def set_default_user(profile, user):
     """
     from aiida.manage.configuration import get_config
     config = get_config()
-    profile.default_user = user.email
+    profile.default_user_email = user.email
     config.update_profile(profile)
     config.store()
 

aiida/cmdline/params/options/commands/setup.py

@@ -42,14 +42,18 @@ def get_profile_attribute_default(attribute_tuple, ctx):
     :return: profile attribute default value if set, or None
     """
     attribute, default = attribute_tuple
+    parts = attribute.split('.')
 
     try:
         validate_profile_parameter(ctx)
     except click.BadParameter:
         return default
     else:
         try:
-            return getattr(ctx.params['profile'], attribute)
+            data = ctx.params['profile'].dictionary
+            for part in parts:
+                data = data[part]
+            return data
         except KeyError:
             return default
 
@@ -140,8 +144,8 @@ def get_quicksetup_password(ctx, param, value):  # pylint: disable=unused-argume
     config = get_config()
 
     for available_profile in config.profiles:
-        if available_profile.database_username == username:
-            value = available_profile.database_password
+        if available_profile.storage_config['database_username'] == username:
+            value = available_profile.storage_config['database_password']
             break
     else:
         value = get_random_string(16)
@@ -248,46 +252,52 @@ def get_quicksetup_password(ctx, param, value):  # pylint: disable=unused-argume
 
 SETUP_DATABASE_ENGINE = QUICKSETUP_DATABASE_ENGINE.clone(
     prompt='Database engine',
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_engine', 'postgresql_psycopg2')),
+    contextual_default=functools.partial(
+        get_profile_attribute_default, ('storage_config.database_engine', 'postgresql_psycopg2')
+    ),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_BACKEND = QUICKSETUP_DATABASE_BACKEND.clone(
     prompt='Database backend',
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_backend', BACKEND_DJANGO)),
+    contextual_default=functools.partial(get_profile_attribute_default, ('storage_backend', BACKEND_DJANGO)),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_HOSTNAME = QUICKSETUP_DATABASE_HOSTNAME.clone(
     prompt='Database host',
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_hostname', 'localhost')),
+    contextual_default=functools.partial(
+        get_profile_attribute_default, ('storage_config.database_hostname', 'localhost')
+    ),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_PORT = QUICKSETUP_DATABASE_PORT.clone(
     prompt='Database port',
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_port', DEFAULT_DBINFO['port'])),
+    contextual_default=functools.partial(
+        get_profile_attribute_default, ('storage_config.database_port', DEFAULT_DBINFO['port'])
+    ),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_NAME = QUICKSETUP_DATABASE_NAME.clone(
     prompt='Database name',
     required=True,
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_name', None)),
+    contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_name', None)),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_USERNAME = QUICKSETUP_DATABASE_USERNAME.clone(
     prompt='Database username',
     required=True,
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_username', None)),
+    contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_username', None)),
     cls=options.interactive.InteractiveOption
 )
 
 SETUP_DATABASE_PASSWORD = QUICKSETUP_DATABASE_PASSWORD.clone(
     prompt='Database password',
     required=True,
-    contextual_default=functools.partial(get_profile_attribute_default, ('database_password', None)),
+    contextual_default=functools.partial(get_profile_attribute_default, ('storage_config.database_password', None)),
     cls=options.interactive.InteractiveOption
 )
 

aiida/cmdline/params/types/profile.py

@@ -49,7 +49,7 @@ def convert(self, value, param, ctx):
                 self.fail(str(exception))
 
             # Create a new empty profile
-            profile = Profile(value, {})
+            profile = Profile(value, {}, validate=False)
         else:
             if self._cannot_exist:
                 self.fail(str(f'the profile `{value}` already exists'))

aiida/manage/configuration/__init__.py

@@ -51,6 +51,7 @@
 
 import os
 import shutil
+from typing import Optional
 import warnings
 
 from aiida.common.warnings import AiidaDeprecationWarning
@@ -119,16 +120,14 @@ def check_version():
         echo.echo_warning('as you might not be able to automatically migrate your data.\n')
 
 
-def load_profile(profile=None):
+def load_profile(profile: Optional[str] = None) -> Profile:
     """Load a profile.
 
     .. note:: if a profile is already loaded and no explicit profile is specified, nothing will be done
 
     :param profile: the name of the profile to load, by default will use the one marked as default in the config
-    :type profile: str
 
     :return: the loaded `Profile` instance
-    :rtype: :class:`~aiida.manage.configuration.Profile`
     :raises `aiida.common.exceptions.InvalidOperation`: if the backend of another profile has already been loaded
     """
     from aiida.common import InvalidOperation
@@ -235,11 +234,10 @@ def _merge_deprecated_cache_yaml(config, filepath):
     shutil.move(cache_path, cache_path_backup)
 
 
-def get_profile():
+def get_profile() -> Profile:
     """Return the currently loaded profile.
 
     :return: the globally loaded `Profile` instance or `None`
-    :rtype: :class:`~aiida.manage.configuration.Profile`
     """
     global PROFILE  # pylint: disable=global-variable-not-assigned
     return PROFILE
@@ -358,17 +356,25 @@ def load_documentation_profile():
 
     with tempfile.NamedTemporaryFile() as handle:
         profile_name = 'readthedocs'
-        profile = {
-            'AIIDADB_ENGINE': 'postgresql_psycopg2',
-            'AIIDADB_BACKEND': 'django',
-            'AIIDADB_PORT': 5432,
-            'AIIDADB_HOST': 'localhost',
-            'AIIDADB_NAME': 'aiidadb',
-            'AIIDADB_PASS': 'aiidadb',
-            'AIIDADB_USER': 'aiida',
-            'AIIDADB_REPOSITORY_URI': 'file:///dev/null',
+        profile_config = {
+            'storage_backend': 'django',
+            'storage_config': {
+                'database_engine': 'postgresql_psycopg2',
+                'database_port': 5432,
+                'database_hostname': 'localhost',
+                'database_name': 'aiidadb',
+                'database_password': 'aiidadb',
+                'database_username': 'aiida',
+                'repository_uri': 'file:///dev/null',
+            },
+            'broker_protocol': 'amqp',
+            'broker_username': 'guest',
+            'broker_password': 'guest',
+            'broker_host': 'localhost',
+            'broker_port': 5672,
+            'broker_virtual_host': '',
         }
-        config = {'default_profile': profile_name, 'profiles': {profile_name: profile}}
-        PROFILE = Profile(profile_name, profile, from_config=True)
+        config = {'default_profile': profile_name, 'profiles': {profile_name: profile_config}}
+        PROFILE = Profile(profile_name, profile_config)
         CONFIG = Config(handle.name, config)
         get_manager()._load_backend(schema_check=False, repository_check=False)  # pylint: disable=protected-access

aiida/manage/configuration/config.py

@@ -26,7 +26,7 @@
 
 __all__ = ('Config', 'config_schema', 'ConfigValidationError')
 
-SCHEMA_FILE = 'config-v5.schema.json'
+SCHEMA_FILE = 'config-v6.schema.json'
 
 
 @lru_cache(1)
@@ -173,9 +173,7 @@ def __init__(self, filepath: str, config: dict, validate: bool = True):
             self._default_profile = None
 
         for name, config_profile in config.get(self.KEY_PROFILES, {}).items():
-            if Profile.contains_unknown_keys(config_profile):
-                self.handle_invalid(f'encountered unknown keys in profile `{name}` which have been removed')
-            self._profiles[name] = Profile(name, config_profile, from_config=True)
+            self._profiles[name] = Profile(name, config_profile)
 
     def __eq__(self, other):
         """Two configurations are considered equal, when their dictionaries are equal."""
@@ -294,7 +292,7 @@ def validate_profile(self, name):
         if name not in self.profile_names:
             raise exceptions.ProfileConfigurationError(f'profile `{name}` does not exist')
 
-    def get_profile(self, name=None):
+    def get_profile(self, name: Optional[str] = None) -> Profile:
         """Return the profile for the given name or the default one if not specified.
 
         :return: the profile instance or None if it does not exist
@@ -350,12 +348,13 @@ def delete_profile(
         include_database_user: bool = False,
         include_repository: bool = True
     ):
-        """Delete a profile including its contents.
+        """Delete a profile including its storage.
 
         :param include_database: also delete the database configured for the profile.
         :param include_database_user: also delete the database user configured for the profile.
         :param include_repository: also delete the repository configured for the profile.
         """
+        # to-do storage backend specific stuff should be handled by the backend itself
         from aiida.manage.external.postgres import Postgres
 
         profile = self.get_profile(name)
@@ -367,11 +366,11 @@ def delete_profile(
 
         if include_database:
             postgres = Postgres.from_profile(profile)
-            if postgres.db_exists(profile.database_name):
-                postgres.drop_db(profile.database_name)
+            if postgres.db_exists(profile.storage_config['database_name']):
+                postgres.drop_db(profile.storage_config['database_name'])
 
-        if include_database_user and postgres.dbuser_exists(profile.database_username):
-            postgres.drop_dbuser(profile.database_username)
+        if include_database_user and postgres.dbuser_exists(profile.storage_config['database_username']):
+            postgres.drop_dbuser(profile.storage_config['database_username'])
 
         self.remove_profile(name)
         self.store()