Make storage backends pluginnable and dynamically loaded through entry points

.github/config/profile.yaml

@@ -4,7 +4,7 @@ email: aiida@localhost
 first_name: Giuseppe
 last_name: Verdi
 institution: Khedivial
-db_backend: psql_dos
+db_backend: core.psql_dos
 db_engine: postgresql_psycopg2
 db_host: localhost
 db_port: 5432

.molecule/default/config_local.yml

@@ -63,7 +63,7 @@ provisioner:
           aiida_pip_cache: /home/.cache/pip
           venv_bin: /opt/conda/bin
           ansible_python_interpreter: "{{ venv_bin }}/python"
-          aiida_backend: ${AIIDA_TEST_BACKEND:-psql_dos}
+          aiida_backend: ${AIIDA_TEST_BACKEND:-core.psql_dos}
           aiida_workers: ${AIIDA_TEST_WORKERS:-2}
           aiida_path: /tmp/.aiida_${AIIDA_TEST_BACKEND:-psql_dos}
           aiida_query_stats: true

Dockerfile

@@ -9,7 +9,7 @@ ENV USER_EMAIL aiida@localhost
 ENV USER_FIRST_NAME Giuseppe
 ENV USER_LAST_NAME Verdi
 ENV USER_INSTITUTION Khedivial
-ENV AIIDADB_BACKEND psql_dos
+ENV AIIDADB_BACKEND core.psql_dos
 
 # Copy and install AiiDA
 COPY . aiida-core

aiida/cmdline/commands/cmd_devel.py

@@ -99,11 +99,11 @@ def devel_validate_plugins():
 @verdi_devel.command('run-sql')
 @click.argument('sql', type=str)
 def devel_run_sql(sql):
-    """Run a raw SQL command on the profile database (only available for 'psql_dos' storage)."""
+    """Run a raw SQL command on the profile database (only available for 'core.psql_dos' storage)."""
     from sqlalchemy import text
 
     from aiida.storage.psql_dos.utils import create_sqlalchemy_engine
-    assert get_profile().storage_backend == 'psql_dos'
+    assert get_profile().storage_backend == 'core.psql_dos'
     with create_sqlalchemy_engine(get_profile().storage_config).connect() as connection:
         result = connection.execute(text(sql)).fetchall()
 

aiida/cmdline/params/options/commands/setup.py

@@ -259,7 +259,7 @@ def get_quicksetup_password(ctx, param, value):  # pylint: disable=unused-argume
 
 SETUP_DATABASE_BACKEND = QUICKSETUP_DATABASE_BACKEND.clone(
     prompt='Database backend',
-    contextual_default=functools.partial(get_profile_attribute_default, ('storage_backend', 'psql_dos')),
+    contextual_default=functools.partial(get_profile_attribute_default, ('storage_backend', 'core.psql_dos')),
     cls=options.interactive.InteractiveOption
 )
 

aiida/cmdline/params/options/main.py

@@ -282,7 +282,7 @@ def set_log_level(_ctx, _param, value):
 )
 
 DB_BACKEND = OverridableOption(
-    '--db-backend', type=click.Choice(['psql_dos']), default='psql_dos', help='Database backend to use.'
+    '--db-backend', type=click.Choice(['core.psql_dos']), default='core.psql_dos', help='Database backend to use.'
 )
 
 DB_HOST = OverridableOption(

aiida/manage/configuration/__init__.py

@@ -278,7 +278,7 @@ def load_documentation_profile():
         profile_name = 'readthedocs'
         profile_config = {
             'storage': {
-                'backend': 'psql_dos',
+                'backend': 'core.psql_dos',
                 'config': {
                     'database_engine': 'postgresql_psycopg2',
                     'database_port': 5432,

aiida/manage/configuration/config.py

@@ -27,7 +27,7 @@
 
 __all__ = ('Config', 'config_schema', 'ConfigValidationError')
 
-SCHEMA_FILE = 'config-v8.schema.json'
+SCHEMA_FILE = 'config-v9.schema.json'
 
 
 @lru_cache(1)

aiida/manage/configuration/migrations/migrations.py

@@ -25,8 +25,8 @@
 # When the configuration file format is changed in a backwards-incompatible way, the oldest compatible version should
 # be set to the new current version.
 
-CURRENT_CONFIG_VERSION = 8
-OLDEST_COMPATIBLE_CONFIG_VERSION = 8
+CURRENT_CONFIG_VERSION = 9
+OLDEST_COMPATIBLE_CONFIG_VERSION = 9
 
 CONFIG_LOGGER = AIIDA_LOGGER.getChild('config')
 
@@ -344,6 +344,38 @@ def downgrade(self, config: ConfigType) -> None:
             profiles[profile_name] = profile
 
 
+class AddPrefixToStorageBackendTypes(SingleMigration):
+    """The ``storage.backend`` key should be prefixed with ``core.``.
+
+    At this point, it should only ever contain ``psql_dos`` which should therefore become ``core.psql_dos``. To cover
+    for cases where people manually added a read only ``sqlite_zip`` profile, we also migrate that.
+    """
+    down_revision = 8
+    down_compatible = 8
+    up_revision = 9
+    up_compatible = 9
+
+    def upgrade(self, config: ConfigType) -> None:
+        for profile_name, profile in config.get('profiles', {}).items():
+            if 'storage' in profile:
+                backend = profile['storage'].get('backend', None)
+                if backend in ('psql_dos', 'sqlite_zip', 'sqlite_temp'):
+                    profile['storage']['backend'] = 'core.' + backend
+                else:
+                    CONFIG_LOGGER.warning(f'profile {profile_name!r} had unknown storage backend {backend!r}')
+
+    def downgrade(self, config: ConfigType) -> None:
+        for profile_name, profile in config.get('profiles', {}).items():
+            backend = profile.get('storage', {}).get('backend', None)
+            if backend in ('core.psql_dos', 'core.sqlite_zip', 'core.sqlite_temp'):
+                profile.setdefault('storage', {})['backend'] = backend[5:]
+            else:
+                CONFIG_LOGGER.warning(
+                    f'profile {profile_name!r} has storage backend {backend!r} that will not be compatible '
+                    'with the version of `aiida-core` that can be used with the new version of the configuration.'
+                )
+
+
 MIGRATIONS = (
     Initial,
     AddProfileUuid,
@@ -353,6 +385,7 @@ def downgrade(self, config: ConfigType) -> None:
     AbstractStorageAndProcess,
     MergeStorageBackendTypes,
     AddTestProfileKey,
+    AddPrefixToStorageBackendTypes,
 )
 
 

aiida/manage/configuration/profile.py

@@ -124,16 +124,8 @@ def set_storage(self, name: str, config: Dict[str, Any]) -> None:
     @property
     def storage_cls(self) -> Type['StorageBackend']:
         """Return the storage backend class for this profile."""
-        if self.storage_backend == 'psql_dos':
-            from aiida.storage.psql_dos.backend import PsqlDosBackend
-            return PsqlDosBackend
-        if self.storage_backend == 'sqlite_zip':
-            from aiida.storage.sqlite_zip.backend import SqliteZipBackend
-            return SqliteZipBackend
-        if self.storage_backend == 'sqlite_temp':
-            from aiida.storage.sqlite_temp.backend import SqliteTempBackend
-            return SqliteTempBackend
-        raise ValueError(f'unknown storage backend type: {self.storage_backend}')
+        from aiida.plugins import StorageFactory
+        return StorageFactory(self.storage_backend)
 
     @property
     def process_control_backend(self) -> str: