From fdb0e9d57ec7dc5e7140e22a1a86e54c10329c62 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Fri, 22 Apr 2022 16:54:33 +0200 Subject: [PATCH 1/3] Add the `aiida.storage` entry point group This will allow plugins to dynamically provide custom storage backend implementations. Each profile already defines the class to be used in the `storage.backend` key, however, so far the conversion of that string identifier to the corresponding class was hard coded. Here we add the `aiida.storage` entry point group that registers the two storage backend implementations currently provided by `aiida-core`, namely, `psql_dos` and `sqlite_zip`. The `Profile.storage_cls` property now treats the `storage.backend` key as an entry point and tries to load the corresponding class using the `aiida.plugins.StorageFactory`. --- aiida/manage/configuration/profile.py | 12 ++---------- aiida/plugins/__init__.py | 1 + aiida/plugins/entry_point.py | 1 + aiida/plugins/factories.py | 27 +++++++++++++++++++++++++-- pyproject.toml | 5 +++++ tests/plugins/test_factories.py | 14 ++++++++++++++ 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/aiida/manage/configuration/profile.py b/aiida/manage/configuration/profile.py index cd114a8199..de9ef4a834 100644 --- a/aiida/manage/configuration/profile.py +++ b/aiida/manage/configuration/profile.py @@ -124,16 +124,8 @@ def set_storage(self, name: str, config: Dict[str, Any]) -> None: @property def storage_cls(self) -> Type['StorageBackend']: """Return the storage backend class for this profile.""" - if self.storage_backend == 'psql_dos': - from aiida.storage.psql_dos.backend import PsqlDosBackend - return PsqlDosBackend - if self.storage_backend == 'sqlite_zip': - from aiida.storage.sqlite_zip.backend import SqliteZipBackend - return SqliteZipBackend - if self.storage_backend == 'sqlite_temp': - from aiida.storage.sqlite_temp.backend import SqliteTempBackend - return SqliteTempBackend - raise ValueError(f'unknown storage backend type: {self.storage_backend}') + from aiida.plugins import StorageFactory + return StorageFactory(self.storage_backend) @property def process_control_backend(self) -> str: diff --git a/aiida/plugins/__init__.py b/aiida/plugins/__init__.py index 5c3e731676..c09ca21af1 100644 --- a/aiida/plugins/__init__.py +++ b/aiida/plugins/__init__.py @@ -29,6 +29,7 @@ 'ParserFactory', 'PluginVersionProvider', 'SchedulerFactory', + 'StorageFactory', 'TransportFactory', 'WorkflowFactory', 'get_entry_points', diff --git a/aiida/plugins/entry_point.py b/aiida/plugins/entry_point.py index d8c67ea741..20a3e9fa6a 100644 --- a/aiida/plugins/entry_point.py +++ b/aiida/plugins/entry_point.py @@ -65,6 +65,7 @@ class EntryPointFormat(enum.Enum): 'aiida.node': 'aiida.orm.nodes', 'aiida.parsers': 'aiida.parsers.plugins', 'aiida.schedulers': 'aiida.schedulers.plugins', + 'aiida.storage': 'aiida.storage', 'aiida.tools.calculations': 'aiida.tools.calculations', 'aiida.tools.data.orbitals': 'aiida.tools.data.orbitals', 'aiida.tools.dbexporters': 'aiida.tools.dbexporters', diff --git a/aiida/plugins/factories.py b/aiida/plugins/factories.py index 18187da4af..b0da92cedd 100644 --- a/aiida/plugins/factories.py +++ b/aiida/plugins/factories.py @@ -18,12 +18,12 @@ __all__ = ( 'BaseFactory', 'CalculationFactory', 'CalcJobImporterFactory', 'DataFactory', 'DbImporterFactory', 'GroupFactory', - 'OrbitalFactory', 'ParserFactory', 'SchedulerFactory', 'TransportFactory', 'WorkflowFactory' + 'OrbitalFactory', 'ParserFactory', 'SchedulerFactory', 'StorageFactory', 'TransportFactory', 'WorkflowFactory' ) if TYPE_CHECKING: from aiida.engine import CalcJob, CalcJobImporter, WorkChain - from aiida.orm import Data, Group + from aiida.orm import Data, Group, StorageBackend from aiida.parsers import Parser from aiida.schedulers import Scheduler from aiida.tools.data.orbital import Orbital @@ -248,6 +248,29 @@ def SchedulerFactory(entry_point_name: str, load: bool = True) -> Optional[Union raise_invalid_type_error(entry_point_name, entry_point_group, valid_classes) +def StorageFactory(entry_point_name: str, load: bool = True) -> Optional[Union[EntryPoint, 'StorageBackend']]: + """Return the ``StorageBackend`` sub class registered under the given entry point. + + :param entry_point_name: the entry point name. + :param load: if True, load the matched entry point and return the loaded resource instead of the entry point itself. + :return: sub class of :py:class:`~aiida.orm.implementation.storage_backend.StorageBackend`. + :raises aiida.common.InvalidEntryPointTypeError: if the type of the loaded entry point is invalid. + """ + from aiida.orm.implementation import StorageBackend + + entry_point_group = 'aiida.storage' + entry_point = BaseFactory(entry_point_group, entry_point_name, load=load) + valid_classes = (StorageBackend,) + + if not load: + return entry_point + + if isclass(entry_point) and issubclass(entry_point, StorageBackend): + return entry_point + + raise_invalid_type_error(entry_point_name, entry_point_group, valid_classes) + + def TransportFactory(entry_point_name: str, load: bool = True) -> Optional[Union[EntryPoint, Type['Transport']]]: """Return the `Transport` sub class registered under the given entry point. diff --git a/pyproject.toml b/pyproject.toml index 3369317427..33f9731913 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -213,6 +213,11 @@ runaiida = "aiida.cmdline.commands.cmd_run:run" "core.slurm" = "aiida.schedulers.plugins.slurm:SlurmScheduler" "core.torque" = "aiida.schedulers.plugins.torque:TorqueScheduler" +[project.entry-points."aiida.storage"] +"core.psql_dos" = "aiida.storage.psql_dos.backend:PsqlDosBackend" +"core.sqlite_temp" = "aiida.storage.sqlite_temp.backend:SqliteTempBackend" +"core.sqlite_zip" = "aiida.storage.sqlite_zip.backend:SqliteZipBackend" + [project.entry-points."aiida.transports"] "core.local" = "aiida.transports.plugins.local:LocalTransport" "core.ssh" = "aiida.transports.plugins.ssh:SshTransport" diff --git a/tests/plugins/test_factories.py b/tests/plugins/test_factories.py index 037d29ce63..7e71c1737e 100644 --- a/tests/plugins/test_factories.py +++ b/tests/plugins/test_factories.py @@ -14,6 +14,7 @@ from aiida.common.exceptions import InvalidEntryPointTypeError from aiida.engine import CalcJob, CalcJobImporter, WorkChain, calcfunction, workfunction from aiida.orm import CalcFunctionNode, Data, Node, WorkFunctionNode +from aiida.orm.implementation.storage_backend import StorageBackend from aiida.parsers import Parser from aiida.plugins import entry_point, factories from aiida.schedulers import Scheduler @@ -64,6 +65,10 @@ def work_function(): 'valid': Scheduler, 'invalid': Node, }, + 'aiida.storage': { + 'valid': StorageBackend, + 'invalid': Node, + }, 'aiida.transports': { 'valid': Transport, 'invalid': Node, @@ -174,6 +179,15 @@ def test_scheduler_factory(self): with pytest.raises(InvalidEntryPointTypeError): factories.SchedulerFactory('invalid') + @pytest.mark.usefixtures('mock_load_entry_point') + def test_storage_factory(self): + """Test the ``StorageFactory``.""" + plugin = factories.StorageFactory('valid') + assert plugin is StorageBackend + + with pytest.raises(InvalidEntryPointTypeError): + factories.StorageFactory('invalid') + @pytest.mark.usefixtures('mock_load_entry_point') def test_transport_factory(self): """Test the ``TransportFactory``.""" From 227fd21f62ceb4a79300b0c7ce5164935c229794 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Fri, 22 Apr 2022 17:03:13 +0200 Subject: [PATCH 2/3] Add the prefix `core.` to all storage entry points --- .github/config/profile.yaml | 2 +- .molecule/default/config_local.yml | 2 +- Dockerfile | 2 +- aiida/cmdline/commands/cmd_devel.py | 4 ++-- aiida/cmdline/params/options/commands/setup.py | 2 +- aiida/cmdline/params/options/main.py | 2 +- aiida/manage/configuration/__init__.py | 2 +- aiida/manage/tests/main.py | 12 ++++++------ aiida/storage/sqlite_temp/backend.py | 2 +- aiida/storage/sqlite_zip/backend.py | 2 +- docs/source/howto/installation.rst | 4 ++-- docs/source/reference/command_line.rst | 4 ++-- tests/cmdline/commands/test_setup.py | 2 +- tests/conftest.py | 2 +- tests/manage/configuration/test_profile.py | 2 +- tests/storage/psql_dos/conftest.py | 2 +- tests/storage/psql_dos/migrations/conftest.py | 2 +- 17 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/config/profile.yaml b/.github/config/profile.yaml index 84bdab3e91..2f07476462 100644 --- a/.github/config/profile.yaml +++ b/.github/config/profile.yaml @@ -4,7 +4,7 @@ email: aiida@localhost first_name: Giuseppe last_name: Verdi institution: Khedivial -db_backend: psql_dos +db_backend: core.psql_dos db_engine: postgresql_psycopg2 db_host: localhost db_port: 5432 diff --git a/.molecule/default/config_local.yml b/.molecule/default/config_local.yml index 2db8c417f8..7e7eb34736 100644 --- a/.molecule/default/config_local.yml +++ b/.molecule/default/config_local.yml @@ -63,7 +63,7 @@ provisioner: aiida_pip_cache: /home/.cache/pip venv_bin: /opt/conda/bin ansible_python_interpreter: "{{ venv_bin }}/python" - aiida_backend: ${AIIDA_TEST_BACKEND:-psql_dos} + aiida_backend: ${AIIDA_TEST_BACKEND:-core.psql_dos} aiida_workers: ${AIIDA_TEST_WORKERS:-2} aiida_path: /tmp/.aiida_${AIIDA_TEST_BACKEND:-psql_dos} aiida_query_stats: true diff --git a/Dockerfile b/Dockerfile index d53529f6eb..698907ee84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV USER_EMAIL aiida@localhost ENV USER_FIRST_NAME Giuseppe ENV USER_LAST_NAME Verdi ENV USER_INSTITUTION Khedivial -ENV AIIDADB_BACKEND psql_dos +ENV AIIDADB_BACKEND core.psql_dos # Copy and install AiiDA COPY . aiida-core diff --git a/aiida/cmdline/commands/cmd_devel.py b/aiida/cmdline/commands/cmd_devel.py index fdf62f8a10..277fbd07b8 100644 --- a/aiida/cmdline/commands/cmd_devel.py +++ b/aiida/cmdline/commands/cmd_devel.py @@ -99,11 +99,11 @@ def devel_validate_plugins(): @verdi_devel.command('run-sql') @click.argument('sql', type=str) def devel_run_sql(sql): - """Run a raw SQL command on the profile database (only available for 'psql_dos' storage).""" + """Run a raw SQL command on the profile database (only available for 'core.psql_dos' storage).""" from sqlalchemy import text from aiida.storage.psql_dos.utils import create_sqlalchemy_engine - assert get_profile().storage_backend == 'psql_dos' + assert get_profile().storage_backend == 'core.psql_dos' with create_sqlalchemy_engine(get_profile().storage_config).connect() as connection: result = connection.execute(text(sql)).fetchall() diff --git a/aiida/cmdline/params/options/commands/setup.py b/aiida/cmdline/params/options/commands/setup.py index 13132b95e7..e67c517864 100644 --- a/aiida/cmdline/params/options/commands/setup.py +++ b/aiida/cmdline/params/options/commands/setup.py @@ -259,7 +259,7 @@ def get_quicksetup_password(ctx, param, value): # pylint: disable=unused-argume SETUP_DATABASE_BACKEND = QUICKSETUP_DATABASE_BACKEND.clone( prompt='Database backend', - contextual_default=functools.partial(get_profile_attribute_default, ('storage_backend', 'psql_dos')), + contextual_default=functools.partial(get_profile_attribute_default, ('storage_backend', 'core.psql_dos')), cls=options.interactive.InteractiveOption ) diff --git a/aiida/cmdline/params/options/main.py b/aiida/cmdline/params/options/main.py index 1125b66ec1..c33ed7bb45 100644 --- a/aiida/cmdline/params/options/main.py +++ b/aiida/cmdline/params/options/main.py @@ -282,7 +282,7 @@ def set_log_level(_ctx, _param, value): ) DB_BACKEND = OverridableOption( - '--db-backend', type=click.Choice(['psql_dos']), default='psql_dos', help='Database backend to use.' + '--db-backend', type=click.Choice(['core.psql_dos']), default='core.psql_dos', help='Database backend to use.' ) DB_HOST = OverridableOption( diff --git a/aiida/manage/configuration/__init__.py b/aiida/manage/configuration/__init__.py index 3ef907d78b..11ca28d3a0 100644 --- a/aiida/manage/configuration/__init__.py +++ b/aiida/manage/configuration/__init__.py @@ -278,7 +278,7 @@ def load_documentation_profile(): profile_name = 'readthedocs' profile_config = { 'storage': { - 'backend': 'psql_dos', + 'backend': 'core.psql_dos', 'config': { 'database_engine': 'postgresql_psycopg2', 'database_port': 5432, diff --git a/aiida/manage/tests/main.py b/aiida/manage/tests/main.py index be5da4efd2..586b479898 100644 --- a/aiida/manage/tests/main.py +++ b/aiida/manage/tests/main.py @@ -38,7 +38,7 @@ 'first_name': 'AiiDA', 'last_name': 'Plugintest', 'institution': 'aiidateam', - 'storage_backend': 'psql_dos', + 'storage_backend': 'core.psql_dos', 'database_engine': 'postgresql_psycopg2', 'database_username': 'aiida', 'database_password': 'aiida_pw', @@ -219,7 +219,7 @@ class TemporaryProfileManager(ProfileManager): """ - def __init__(self, backend='psql_dos', pgtest=None): # pylint: disable=super-init-not-called + def __init__(self, backend='core.psql_dos', pgtest=None): # pylint: disable=super-init-not-called """Construct a TemporaryProfileManager :param backend: a database backend @@ -368,7 +368,7 @@ def backend(self, backend): if self.has_profile_open(): raise TestManagerError('backend cannot be changed after setting up the environment') - valid_backends = ['psql_dos'] + valid_backends = ['core.psql_dos'] if backend not in valid_backends: raise ValueError(f'invalid backend {backend}, must be one of {valid_backends}') self.profile_info['backend'] = backend @@ -418,7 +418,7 @@ def has_profile_open(self): @contextmanager -def test_manager(backend='psql_dos', profile_name=None, pgtest=None): +def test_manager(backend='core.psql_dos', profile_name=None, pgtest=None): """ Context manager for TestManager objects. Sets up temporary AiiDA environment for testing or reuses existing environment, @@ -480,9 +480,9 @@ def get_test_backend_name() -> str: ) backend_res = backend_profile else: - backend_res = backend_env or 'psql_dos' + backend_res = backend_env or 'core.psql_dos' - if backend_res in ('psql_dos',): + if backend_res in ('core.psql_dos',): return backend_res raise ValueError(f"Unknown backend '{backend_res}' read from AIIDA_TEST_BACKEND environment variable") diff --git a/aiida/storage/sqlite_temp/backend.py b/aiida/storage/sqlite_temp/backend.py index d6f71249be..001785c3dd 100644 --- a/aiida/storage/sqlite_temp/backend.py +++ b/aiida/storage/sqlite_temp/backend.py @@ -55,7 +55,7 @@ def create_profile( { 'default_user_email': default_user_email, 'storage': { - 'backend': 'sqlite_temp', + 'backend': 'core.sqlite_temp', 'config': { 'debug': debug, # Note this is currently required, see https://github.com/aiidateam/aiida-core/issues/5451 diff --git a/aiida/storage/sqlite_zip/backend.py b/aiida/storage/sqlite_zip/backend.py index deb8266dfe..394ce8e8f6 100644 --- a/aiida/storage/sqlite_zip/backend.py +++ b/aiida/storage/sqlite_zip/backend.py @@ -61,7 +61,7 @@ def create_profile(path: str | Path, options: dict | None = None) -> Profile: return Profile( profile_name, { 'storage': { - 'backend': 'sqlite_zip', + 'backend': 'core.sqlite_zip', 'config': { 'path': str(path) } diff --git a/docs/source/howto/installation.rst b/docs/source/howto/installation.rst index 3baeb4c97f..0bf7bca3a2 100644 --- a/docs/source/howto/installation.rst +++ b/docs/source/howto/installation.rst @@ -53,7 +53,7 @@ To display these parameters, use ``verdi profile show``: broker_username: guest broker_virtual_host: '' storage: - backend: psql_dos + backend: core.psql_dos config: database_engine: postgresql_psycopg2 database_hostname: localhost @@ -568,7 +568,7 @@ To determine what storage backend a profile uses, call ``verdi profile show``. .. tab-item:: psql_dos - To fully backup the data stored for a profile using the ``psql_dos`` backend, you should restore the associated database and file repository. + To fully backup the data stored for a profile using the ``core.psql_dos`` backend, you should restore the associated database and file repository. **PostgreSQL database** diff --git a/docs/source/reference/command_line.rst b/docs/source/reference/command_line.rst index 0390b7868b..4a1adab63e 100644 --- a/docs/source/reference/command_line.rst +++ b/docs/source/reference/command_line.rst @@ -377,7 +377,7 @@ Below is a list with all available subcommands. --institution NONEMPTYSTRING Institution of the user. [required] --db-engine [postgresql_psycopg2] Engine to use to connect to the database. - --db-backend [psql_dos] Database backend to use. + --db-backend [core.psql_dos] Database backend to use. --db-host HOSTNAME Database server host. Leave empty for "peer" authentication. --db-port INTEGER Database server port. @@ -481,7 +481,7 @@ Below is a list with all available subcommands. --institution NONEMPTYSTRING Institution of the user. [required] --db-engine [postgresql_psycopg2] Engine to use to connect to the database. - --db-backend [psql_dos] Database backend to use. + --db-backend [core.psql_dos] Database backend to use. --db-host HOSTNAME Database server host. Leave empty for "peer" authentication. --db-port INTEGER Database server port. diff --git a/tests/cmdline/commands/test_setup.py b/tests/cmdline/commands/test_setup.py index fdf01df07e..dca102223e 100644 --- a/tests/cmdline/commands/test_setup.py +++ b/tests/cmdline/commands/test_setup.py @@ -35,7 +35,7 @@ class TestVerdiSetup: def init_profile(self, pg_test_cluster, empty_config, run_cli_command): # pylint: disable=redefined-outer-name,unused-argument """Initialize the profile.""" # pylint: disable=attribute-defined-outside-init - self.storage_backend_name = 'psql_dos' + self.storage_backend_name = 'core.psql_dos' self.pg_test = pg_test_cluster self.cli_runner = run_cli_command diff --git a/tests/conftest.py b/tests/conftest.py index 6745b4fe5f..f9550f70f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -240,7 +240,7 @@ def _create_profile(name='test-profile', **kwargs): profile_dictionary = { 'default_user_email': kwargs.pop('default_user_email', 'dummy@localhost'), 'storage': { - 'backend': kwargs.pop('storage_backend', 'psql_dos'), + 'backend': kwargs.pop('storage_backend', 'core.psql_dos'), 'config': { 'database_engine': kwargs.pop('database_engine', 'postgresql_psycopg2'), 'database_hostname': kwargs.pop('database_hostname', 'localhost'), diff --git a/tests/manage/configuration/test_profile.py b/tests/manage/configuration/test_profile.py index 245bf6a632..3a8c25aab9 100644 --- a/tests/manage/configuration/test_profile.py +++ b/tests/manage/configuration/test_profile.py @@ -17,7 +17,7 @@ def test_base_properties(profile_factory): """Test the basic properties of a ``Profile`` instance.""" kwargs = { 'name': 'profile-name', - 'storage_backend': 'psql_dos', + 'storage_backend': 'core.psql_dos', 'process_control_backend': 'rabbitmq', } profile = profile_factory(**kwargs) diff --git a/tests/storage/psql_dos/conftest.py b/tests/storage/psql_dos/conftest.py index 7faf903e80..cbad5c9c83 100644 --- a/tests/storage/psql_dos/conftest.py +++ b/tests/storage/psql_dos/conftest.py @@ -10,5 +10,5 @@ """Configuration file for pytest tests.""" from aiida.manage.tests import get_test_backend_name -if get_test_backend_name() != 'psql_dos': +if get_test_backend_name() != 'core.psql_dos': collect_ignore_glob = ['*'] # pylint: disable=invalid-name diff --git a/tests/storage/psql_dos/migrations/conftest.py b/tests/storage/psql_dos/migrations/conftest.py index 347ccdde74..762548f35c 100644 --- a/tests/storage/psql_dos/migrations/conftest.py +++ b/tests/storage/psql_dos/migrations/conftest.py @@ -78,7 +78,7 @@ def uninitialised_profile(empty_pg_cluster: PGTest, tmp_path): # pylint: disabl 'test_migrate', { 'test_profile': True, 'storage': { - 'backend': 'psql_dos', + 'backend': 'core.psql_dos', 'config': { 'database_engine': 'postgresql_psycopg2', 'database_port': empty_pg_cluster.port, From fa871871a6bd091299898dad59bb08eb0b52a059 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Fri, 22 Apr 2022 17:20:04 +0200 Subject: [PATCH 3/3] Config: add migration to properly prefix storage backend Existing profiles should have the key `storage.backend` be prefixed with `core.` to indicate that those entry points ship with `aiida-core`. --- aiida/manage/configuration/config.py | 2 +- .../configuration/migrations/migrations.py | 37 +- .../schema/config-v9.schema.json | 351 ++++++++++++++++++ .../migrations/test_samples/input/8.json | 35 ++ .../migrations/test_samples/reference/9.json | 35 ++ .../test_samples/reference/final.json | 4 +- 6 files changed, 459 insertions(+), 5 deletions(-) create mode 100644 aiida/manage/configuration/schema/config-v9.schema.json create mode 100644 tests/manage/configuration/migrations/test_samples/input/8.json create mode 100644 tests/manage/configuration/migrations/test_samples/reference/9.json diff --git a/aiida/manage/configuration/config.py b/aiida/manage/configuration/config.py index 4b444ac288..6389b1ee2c 100644 --- a/aiida/manage/configuration/config.py +++ b/aiida/manage/configuration/config.py @@ -27,7 +27,7 @@ __all__ = ('Config', 'config_schema', 'ConfigValidationError') -SCHEMA_FILE = 'config-v8.schema.json' +SCHEMA_FILE = 'config-v9.schema.json' @lru_cache(1) diff --git a/aiida/manage/configuration/migrations/migrations.py b/aiida/manage/configuration/migrations/migrations.py index 44eff76d87..6e0c9580ef 100644 --- a/aiida/manage/configuration/migrations/migrations.py +++ b/aiida/manage/configuration/migrations/migrations.py @@ -25,8 +25,8 @@ # When the configuration file format is changed in a backwards-incompatible way, the oldest compatible version should # be set to the new current version. -CURRENT_CONFIG_VERSION = 8 -OLDEST_COMPATIBLE_CONFIG_VERSION = 8 +CURRENT_CONFIG_VERSION = 9 +OLDEST_COMPATIBLE_CONFIG_VERSION = 9 CONFIG_LOGGER = AIIDA_LOGGER.getChild('config') @@ -344,6 +344,38 @@ def downgrade(self, config: ConfigType) -> None: profiles[profile_name] = profile +class AddPrefixToStorageBackendTypes(SingleMigration): + """The ``storage.backend`` key should be prefixed with ``core.``. + + At this point, it should only ever contain ``psql_dos`` which should therefore become ``core.psql_dos``. To cover + for cases where people manually added a read only ``sqlite_zip`` profile, we also migrate that. + """ + down_revision = 8 + down_compatible = 8 + up_revision = 9 + up_compatible = 9 + + def upgrade(self, config: ConfigType) -> None: + for profile_name, profile in config.get('profiles', {}).items(): + if 'storage' in profile: + backend = profile['storage'].get('backend', None) + if backend in ('psql_dos', 'sqlite_zip', 'sqlite_temp'): + profile['storage']['backend'] = 'core.' + backend + else: + CONFIG_LOGGER.warning(f'profile {profile_name!r} had unknown storage backend {backend!r}') + + def downgrade(self, config: ConfigType) -> None: + for profile_name, profile in config.get('profiles', {}).items(): + backend = profile.get('storage', {}).get('backend', None) + if backend in ('core.psql_dos', 'core.sqlite_zip', 'core.sqlite_temp'): + profile.setdefault('storage', {})['backend'] = backend[5:] + else: + CONFIG_LOGGER.warning( + f'profile {profile_name!r} has storage backend {backend!r} that will not be compatible ' + 'with the version of `aiida-core` that can be used with the new version of the configuration.' + ) + + MIGRATIONS = ( Initial, AddProfileUuid, @@ -353,6 +385,7 @@ def downgrade(self, config: ConfigType) -> None: AbstractStorageAndProcess, MergeStorageBackendTypes, AddTestProfileKey, + AddPrefixToStorageBackendTypes, ) diff --git a/aiida/manage/configuration/schema/config-v9.schema.json b/aiida/manage/configuration/schema/config-v9.schema.json new file mode 100644 index 0000000000..9ba239d267 --- /dev/null +++ b/aiida/manage/configuration/schema/config-v9.schema.json @@ -0,0 +1,351 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "Schema for AiiDA configuration files, format version 8", + "type": "object", + "definitions": { + "options": { + "type": "object", + "properties": { + "runner.poll.interval": { + "type": "integer", + "default": 60, + "minimum": 0, + "description": "Polling interval in seconds to be used by process runners" + }, + "daemon.default_workers": { + "type": "integer", + "default": 1, + "minimum": 1, + "description": "Default number of workers to be launched by `verdi daemon start`" + }, + "daemon.timeout": { + "type": "integer", + "default": 20, + "minimum": 0, + "description": "Timeout in seconds for calls to the circus client" + }, + "daemon.worker_process_slots": { + "type": "integer", + "default": 200, + "minimum": 1, + "description": "Maximum number of concurrent process tasks that each daemon worker can handle" + }, + "db.batch_size": { + "type": "integer", + "default": 100000, + "minimum": 1, + "description": "Batch size for bulk CREATE operations in the database. Avoids hitting MaxAllocSize of PostgreSQL (1GB) when creating large numbers of database records in one go." + }, + "verdi.shell.auto_import": { + "type": "string", + "default": "", + "description": "Additional modules/functions/classes to be automatically loaded in `verdi shell`, split by ':'" + }, + "logging.aiida_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "REPORT", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `aiida` logger" + }, + "logging.db_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "REPORT", + "description": "Minimum level to log to the DbLog table" + }, + "logging.plumpy_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `plumpy` logger" + }, + "logging.kiwipy_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `kiwipy` logger" + }, + "logging.paramiko_loglevel": { + "key": "logging_paramiko_log_level", + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `paramiko` logger" + }, + "logging.alembic_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `alembic` logger" + }, + "logging.sqlalchemy_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `sqlalchemy` logger" + }, + "logging.circus_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "INFO", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `circus` logger" + }, + "logging.aiopika_loglevel": { + "type": "string", + "enum": ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"], + "default": "WARNING", + "description": "Minimum level to log to daemon log and the `DbLog` table for the `aio_pika` logger" + }, + "warnings.showdeprecations": { + "type": "boolean", + "default": true, + "description": "Whether to print AiiDA deprecation warnings" + }, + "warnings.development_version": { + "type": "boolean", + "default": true, + "description": "Whether to print a warning when a profile is loaded while a development version is installed", + "global_only": true + }, + "warnings.rabbitmq_version": { + "type": "boolean", + "default": true, + "description": "Whether to print a warning when an incompatible version of RabbitMQ is configured" + }, + "transport.task_retry_initial_interval": { + "type": "integer", + "default": 20, + "minimum": 1, + "description": "Initial time interval for the exponential backoff mechanism." + }, + "transport.task_maximum_attempts": { + "type": "integer", + "default": 5, + "minimum": 1, + "description": "Maximum number of transport task attempts before a Process is Paused." + }, + "rmq.task_timeout": { + "type": "integer", + "default": 10, + "minimum": 1, + "description": "Timeout in seconds for communications with RabbitMQ" + }, + "storage.sandbox": { + "type": "string", + "description": "Absolute path to the directory to store sandbox folders." + }, + "caching.default_enabled": { + "type": "boolean", + "default": false, + "description": "Enable calculation caching by default" + }, + "caching.enabled_for": { + "description": "Calculation entry points to enable caching on", + "type": "array", + "default": [], + "items": { + "type": "string" + } + }, + "caching.disabled_for": { + "description": "Calculation entry points to disable caching on", + "type": "array", + "default": [], + "items": { + "type": "string" + } + }, + "autofill.user.email": { + "type": "string", + "global_only": true, + "description": "Default user email to use when creating new profiles." + }, + "autofill.user.first_name": { + "type": "string", + "global_only": true, + "description": "Default user first name to use when creating new profiles." + }, + "autofill.user.last_name": { + "type": "string", + "global_only": true, + "description": "Default user last name to use when creating new profiles." + }, + "autofill.user.institution": { + "type": "string", + "global_only": true, + "description": "Default user institution to use when creating new profiles." + } + } + }, + "profile": { + "type": "object", + "required": ["storage", "process_control"], + "properties": { + "PROFILE_UUID": { + "description": "The profile's unique key", + "type": "string" + }, + "storage": { + "description": "The storage configuration", + "type": "object", + "required": ["backend", "config"], + "properties": { + "backend": { + "description": "The storage backend type to use", + "type": "string", + "default": "psql_dos" + }, + "config": { + "description": "The configuration to pass to the storage backend", + "type": "object", + "properties": { + "database_engine": { + "type": "string", + "default": "postgresql_psycopg2" + }, + "database_port": { + "type": ["integer", "string"], + "minimum": 1, + "pattern": "\\d+", + "default": 5432 + }, + "database_hostname": { + "type": ["string", "null"], + "default": null + }, + "database_username": { + "type": "string" + }, + "database_password": { + "type": ["string", "null"], + "default": null + }, + "database_name": { + "type": "string" + }, + "repository_uri": { + "description": "URI to the AiiDA object store", + "type": "string" + } + } + } + } + }, + "process_control": { + "description": "The process control configuration", + "type": "object", + "required": ["backend", "config"], + "properties": { + "backend": { + "description": "The process execution backend type to use", + "type": "string", + "default": "rabbitmq" + }, + "config": { + "description": "The configuration to pass to the process execution backend", + "type": "object", + "parameters": { + "broker_protocol": { + "description": "Protocol for connecting to the RabbitMQ server", + "type": "string", + "enum": ["amqp", "amqps"], + "default": "amqp" + }, + "broker_username": { + "description": "Username for RabbitMQ authentication", + "type": "string", + "default": "guest" + }, + "broker_password": { + "description": "Password for RabbitMQ authentication", + "type": "string", + "default": "guest" + }, + "broker_host": { + "description": "Hostname of the RabbitMQ server", + "type": "string", + "default": "127.0.0.1" + }, + "broker_port": { + "description": "Port of the RabbitMQ server", + "type": "integer", + "minimum": 1, + "default": 5672 + }, + "broker_virtual_host": { + "description": "RabbitMQ virtual host to connect to", + "type": "string", + "default": "" + }, + "broker_parameters": { + "description": "RabbitMQ arguments that will be encoded as query parameters", + "type": "object", + "default": { + "heartbeat": 600 + }, + "properties": { + "heartbeat": { + "description": "After how many seconds the peer TCP connection should be considered unreachable", + "type": "integer", + "default": 600, + "minimum": 0 + } + } + } + } + } + } + }, + "default_user_email": { + "type": ["string", "null"], + "default": null + }, + "test_profile": { + "type": "boolean", + "default": false + }, + "options": { + "description": "Profile specific options", + "$ref": "#/definitions/options" + } + } + } + }, + "required": [], + "properties": { + "CONFIG_VERSION": { + "description": "The configuration version", + "type": "object", + "required": ["CURRENT", "OLDEST_COMPATIBLE"], + "properties": { + "CURRENT": { + "description": "Version number of configuration file format", + "type": "integer", + "const": 9 + }, + "OLDEST_COMPATIBLE": { + "description": "Version number of oldest configuration file format this file is compatible with", + "type": "integer", + "const": 9 + } + } + }, + "profiles": { + "description": "Configured profiles", + "type": "object", + "patternProperties": { + ".+": { + "$ref": "#/definitions/profile" + } + } + }, + "default_profile": { + "description": "Default profile to use", + "type": "string" + }, + "options": { + "description": "Global options", + "$ref": "#/definitions/options" + } + } +} diff --git a/tests/manage/configuration/migrations/test_samples/input/8.json b/tests/manage/configuration/migrations/test_samples/input/8.json new file mode 100644 index 0000000000..5ca153216a --- /dev/null +++ b/tests/manage/configuration/migrations/test_samples/input/8.json @@ -0,0 +1,35 @@ +{ + "CONFIG_VERSION": { "CURRENT": 8, "OLDEST_COMPATIBLE": 8 }, + "default_profile": "default", + "profiles": { + "default": { + "default_user_email": "email@aiida.net", + "PROFILE_UUID": "00000000000000000000000000000000", + "storage": { + "backend": "psql_dos", + "config": { + "database_engine": "postgresql_psycopg2", + "database_password": "some_random_password", + "database_name": "aiidadb_qs_some_user", + "database_hostname": "localhost", + "database_port": "5432", + "database_username": "aiida_qs_greschd", + "repository_uri": "file:////home/some_user/.aiida/repository-quicksetup/" + }, + "_v6_backend": "django" + }, + "process_control": { + "backend": "rabbitmq", + "config": { + "broker_protocol": "amqp", + "broker_username": "guest", + "broker_password": "guest", + "broker_host": "127.0.0.1", + "broker_port": 5672, + "broker_virtual_host": "" + } + }, + "test_profile": false + } + } +} diff --git a/tests/manage/configuration/migrations/test_samples/reference/9.json b/tests/manage/configuration/migrations/test_samples/reference/9.json new file mode 100644 index 0000000000..134f0dcc0c --- /dev/null +++ b/tests/manage/configuration/migrations/test_samples/reference/9.json @@ -0,0 +1,35 @@ +{ + "CONFIG_VERSION": { "CURRENT": 9, "OLDEST_COMPATIBLE": 9 }, + "default_profile": "default", + "profiles": { + "default": { + "default_user_email": "email@aiida.net", + "PROFILE_UUID": "00000000000000000000000000000000", + "storage": { + "backend": "core.psql_dos", + "config": { + "database_engine": "postgresql_psycopg2", + "database_password": "some_random_password", + "database_name": "aiidadb_qs_some_user", + "database_hostname": "localhost", + "database_port": "5432", + "database_username": "aiida_qs_greschd", + "repository_uri": "file:////home/some_user/.aiida/repository-quicksetup/" + }, + "_v6_backend": "django" + }, + "process_control": { + "backend": "rabbitmq", + "config": { + "broker_protocol": "amqp", + "broker_username": "guest", + "broker_password": "guest", + "broker_host": "127.0.0.1", + "broker_port": 5672, + "broker_virtual_host": "" + } + }, + "test_profile": false + } + } +} diff --git a/tests/manage/configuration/migrations/test_samples/reference/final.json b/tests/manage/configuration/migrations/test_samples/reference/final.json index 5ca153216a..134f0dcc0c 100644 --- a/tests/manage/configuration/migrations/test_samples/reference/final.json +++ b/tests/manage/configuration/migrations/test_samples/reference/final.json @@ -1,12 +1,12 @@ { - "CONFIG_VERSION": { "CURRENT": 8, "OLDEST_COMPATIBLE": 8 }, + "CONFIG_VERSION": { "CURRENT": 9, "OLDEST_COMPATIBLE": 9 }, "default_profile": "default", "profiles": { "default": { "default_user_email": "email@aiida.net", "PROFILE_UUID": "00000000000000000000000000000000", "storage": { - "backend": "psql_dos", + "backend": "core.psql_dos", "config": { "database_engine": "postgresql_psycopg2", "database_password": "some_random_password",