From 878d156d7ca981d43208eedbbbc2d447a127a93d Mon Sep 17 00:00:00 2001 From: Tim Weber Date: Fri, 27 Dec 2024 19:03:32 +0000 Subject: [PATCH] Bundle migrations and pre-configure Alembic Signed-off-by: Tim Weber --- server/README.md | 9 ++-- server/dearmep/cli/__init__.py | 33 ++++++++++++-- server/dearmep/cli/run_alembic.py | 43 +++++++++++++++++++ server/{ => dearmep/migrations}/alembic.ini | 2 +- server/{ => dearmep}/migrations/env.py | 0 .../{ => dearmep}/migrations/script.py.mako | 0 .../migrations/script.py.mako.license | 0 ...e440490bf_inital_alembic_database_state.py | 0 ...aefe5f0ad_add_scheduler_relevant_tables.py | 0 9 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 server/dearmep/cli/run_alembic.py rename server/{ => dearmep/migrations}/alembic.ini (96%) rename server/{ => dearmep}/migrations/env.py (100%) rename server/{ => dearmep}/migrations/script.py.mako (100%) rename server/{ => dearmep}/migrations/script.py.mako.license (100%) rename server/{ => dearmep}/migrations/versions/20231121-8d7e440490bf_inital_alembic_database_state.py (100%) rename server/{ => dearmep}/migrations/versions/20231214-5d2aefe5f0ad_add_scheduler_relevant_tables.py (100%) diff --git a/server/README.md b/server/README.md index 40d9d860..bf76e137 100644 --- a/server/README.md +++ b/server/README.md @@ -111,9 +111,12 @@ See the data conversion document's section on [converting the audio files](../do DearMEP is using [Alembic](https://alembic.sqlalchemy.org/) to manage the database and perform version upgrades to it. Currently, this is a manual process. +Note that Alembic requires a configuration file telling it where to find the database and migrations. +Fortunately, DearMEP comes with a [prepared `alembic.ini`](dearmep/migrations/alembic.ini) as well as a special CLI subcommand, `dearmep alembic`, that will invoke Alembic with the environment variable `ALEMBIC_CONFIG` already pointing to the location of this configuration file. + ### For Administrators of a DearMEP Instance -If you upgrade DearMEP, make sure to `alembic upgrade head` before starting the `dearmep serve` process again. +If you upgrade DearMEP, make sure to run `dearmep alembic upgrade head` before starting the `dearmep serve` process again. It will ensure that all changes to the database structure required for the new version are applied. Make sure to have a backup of your database, just in case. @@ -122,10 +125,10 @@ Make sure to have a backup of your database, just in case. If you change the models which are reflected in the database, you'll need to do use Alembic to handle database migrations. You can create migrations via ```sh -alembic revision --autogenerate --message "your short alembic message about the reason of this migration" +dearmep alembic revision --autogenerate --message "your short alembic message about the reason of this migration" ``` -Alembic generates a file in `migrations/versions/`. **Check this file** for sanity. You can edit or delete this file though this usually should not be necessary. If you are happy about the migration, commit to version control. +Alembic generates a file in `dearmep/migrations/versions/`. **Check this file** for sanity. You can edit or delete this file though this usually should not be necessary. If you are happy about the migration, commit to version control. Make sure to `upgrade` the database to the latest revision when developing. It is recommended to check the upgrades first, for example by making a copy of the database source and changing the `sqlalchemy.uri` in the `alembic.ini` or by checking the raw SQL with the `--sql` flag. diff --git a/server/dearmep/cli/__init__.py b/server/dearmep/cli/__init__.py index 6a22ec3e..b4f641d3 100644 --- a/server/dearmep/cli/__init__.py +++ b/server/dearmep/cli/__init__.py @@ -8,7 +8,7 @@ import re from argparse import ArgumentParser, Namespace from contextlib import contextmanager -from sys import exit, stderr +from sys import argv, exit, stderr from typing import TYPE_CHECKING from dotenv import load_dotenv @@ -19,7 +19,7 @@ from ..config import CMD_NAME from ..progress import DummyTaskFactory, RichTaskFactory -from . import check, convert, db, dump, importing, serve, version +from . import check, convert, db, dump, importing, run_alembic, serve, version if TYPE_CHECKING: @@ -27,8 +27,15 @@ class Context: - def __init__(self, *, args: Namespace, raw_stdout: bool = False) -> None: + def __init__( + self, + *, + args: Namespace, + parser: ArgumentParser, + raw_stdout: bool = False, + ) -> None: self.args = args + self.parser = parser # Let the Console run on stderr if we need stdout for raw data. self.console = Console(stderr=raw_stdout) self.raw_stdout = raw_stdout @@ -87,17 +94,35 @@ def run() -> None: subparsers = parser.add_subparsers( metavar="COMMAND", ) - for module in (version, dump, serve, db, convert, importing, check): + for module in ( + version, + dump, + serve, + db, + run_alembic, + convert, + importing, + check, + ): module.add_parser( subparsers, help_if_no_subcommand=help_if_no_subcommand, ) help_if_no_subcommand(parser) + + # Special hack for the `alembic` subcommand: Inject a double dash if it's + # used. This is because it needs to pass the rest of argv to Alembic + # verbatim, but argparse can't ignore arguments that look like flags. + # + if argv[1] == "alembic": + argv.insert(2, "--") + args = parser.parse_args() args.func( Context( args=args, + parser=parser, # Commands can opt-in to have a raw stdout. raw_stdout=getattr(args, "raw_stdout", False), ) diff --git a/server/dearmep/cli/run_alembic.py b/server/dearmep/cli/run_alembic.py new file mode 100644 index 00000000..dd2d3efd --- /dev/null +++ b/server/dearmep/cli/run_alembic.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: © 2024 Tim Weber +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +from __future__ import annotations + +import logging +from os import environ +from pathlib import Path +from typing import TYPE_CHECKING, Callable + +import alembic.config + + +if TYPE_CHECKING: + from argparse import ArgumentParser, _SubParsersAction + + from . import Context + + +_logger = logging.getLogger(__name__) + + +CONFIG_VAR = "ALEMBIC_CONFIG" + + +def run(ctx: Context) -> None: + alembic_config = Path(__file__).parent.parent / "migrations/alembic.ini" + environ[CONFIG_VAR] = str(alembic_config) + alembic.config.main(prog=f"{ctx.parser.prog} alembic", argv=ctx.args.args) + + +def add_parser( + subparsers: _SubParsersAction, + help_if_no_subcommand: Callable, # noqa: ARG001 +) -> None: + parser: ArgumentParser = subparsers.add_parser( + "alembic", + help="invoke pre-configured Alembic to execute database migrations", + add_help=False, # don't hijack Alembic's own `-h` + ) + parser.add_argument("args", nargs="*") + parser.set_defaults(func=run) diff --git a/server/alembic.ini b/server/dearmep/migrations/alembic.ini similarity index 96% rename from server/alembic.ini rename to server/dearmep/migrations/alembic.ini index faa4edc9..2e2bcb37 100644 --- a/server/alembic.ini +++ b/server/dearmep/migrations/alembic.ini @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: AGPL-3.0-or-later [alembic] -script_location = ./migrations +script_location = %(here)s file_template = %%(year)d%%(month).2d%%(day).2d-%%(rev)s_%%(slug)s prepend_sys_path = . version_path_separator = os diff --git a/server/migrations/env.py b/server/dearmep/migrations/env.py similarity index 100% rename from server/migrations/env.py rename to server/dearmep/migrations/env.py diff --git a/server/migrations/script.py.mako b/server/dearmep/migrations/script.py.mako similarity index 100% rename from server/migrations/script.py.mako rename to server/dearmep/migrations/script.py.mako diff --git a/server/migrations/script.py.mako.license b/server/dearmep/migrations/script.py.mako.license similarity index 100% rename from server/migrations/script.py.mako.license rename to server/dearmep/migrations/script.py.mako.license diff --git a/server/migrations/versions/20231121-8d7e440490bf_inital_alembic_database_state.py b/server/dearmep/migrations/versions/20231121-8d7e440490bf_inital_alembic_database_state.py similarity index 100% rename from server/migrations/versions/20231121-8d7e440490bf_inital_alembic_database_state.py rename to server/dearmep/migrations/versions/20231121-8d7e440490bf_inital_alembic_database_state.py diff --git a/server/migrations/versions/20231214-5d2aefe5f0ad_add_scheduler_relevant_tables.py b/server/dearmep/migrations/versions/20231214-5d2aefe5f0ad_add_scheduler_relevant_tables.py similarity index 100% rename from server/migrations/versions/20231214-5d2aefe5f0ad_add_scheduler_relevant_tables.py rename to server/dearmep/migrations/versions/20231214-5d2aefe5f0ad_add_scheduler_relevant_tables.py