Merge pull request #1764 from fractal-analytics-platform/1763-create-…

…appropriate-testing-database-based-on-v240

Add clean 2.4.0 database for testing and update GHA

.github/workflows/migrations.yml

@@ -19,7 +19,7 @@ env:
   # with the SOURCE_VERSION of fractal-server.
   # Here we test the script that upgrades the DB from SOURCE_VERSION to the
   #current version.
-  SOURCE_VERSION: 2.0.4
+  SOURCE_VERSION: 2.4.0
   ROOTDIR: tests/data/testing_databases
 
 jobs:

CHANGELOG.md

@@ -1,7 +1,16 @@
 **Note**: Numbers like (\#1234) point to closed Pull Requests on the fractal-server repository.
 
+# Unreleased
+
+* App:
+    * Improve logging in `fractalctl set-db` (\#1764).
+* Testing:
+    * Start tests of migrations from valid v2.4.0 database (\#1764).
+
 # 2.4.1
 
+This is mainly a bugfix release, re-implementing a check that was removed in 2.4.0.
+
 * API:
     * Re-introduce check for existing-user-email in `PATCH /auth/users/{id}/` (\#1760).
 

fractal_server/__main__.py

@@ -88,12 +88,11 @@ def set_db():
     alembic.config.main(argv=alembic_args)
     print("END: alembic.config")
     # Insert default group
-    print("START: Default group creation")
+    print()
     _create_first_group()
-    print("END: Default group creation")
+    print()
     # NOTE: It will be fixed with #1739
     settings = Inject(get_settings)
-    print("START: First user creation")
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
@@ -103,7 +102,7 @@ def set_db():
             is_verified=True,
         )
     )
-    print("END: First user creation")
+    print()
 
 
 def update_db_data():

fractal_server/app/security/__init__.py

@@ -55,9 +55,9 @@
 from fractal_server.app.models import UserGroup
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.schemas.user import UserCreate
-from fractal_server.logger import get_logger
+from fractal_server.logger import set_logger
 
-logger = get_logger(__name__)
+logger = set_logger(__name__)
 
 FRACTAL_DEFAULT_GROUP_NAME = "All"
 
@@ -264,9 +264,10 @@ async def _create_first_user(
         is_verified: `True` if the new user is verifie
         username:
     """
+    function_logger = set_logger("fractal_server.create_first_user")
+    function_logger.info(f"START _create_first_user, with email '{email}'")
     try:
         async with get_async_session_context() as session:
-
             if is_superuser is True:
                 # If a superuser already exists, exit
                 stm = select(UserOAuth).where(  # noqa
@@ -275,9 +276,9 @@ async def _create_first_user(
                 res = await session.execute(stm)
                 existing_superuser = res.scalars().first()
                 if existing_superuser is not None:
-                    logger.info(
-                        f"{existing_superuser.email} superuser already exists,"
-                        f" skip creation of {email}"
+                    function_logger.info(
+                        f"'{existing_superuser.email}' superuser already "
+                        f"exists, skip creation of '{email}'"
                     )
                     return None
 
@@ -292,27 +293,33 @@ async def _create_first_user(
                     if username is not None:
                         kwargs["username"] = username
                     user = await user_manager.create(UserCreate(**kwargs))
-                    logger.info(f"User {user.email} created")
+                    function_logger.info(f"User '{user.email}' created")
 
     except UserAlreadyExists:
-        logger.warning(f"User {email} already exists")
+        function_logger.warning(f"User '{email}' already exists")
+    finally:
+        function_logger.info(f"END   _create_first_user, with email '{email}'")
 
 
 def _create_first_group():
-    logger.info(
-        f"START _create_first_group, with name {FRACTAL_DEFAULT_GROUP_NAME}"
+    function_logger = set_logger("fractal_server.create_first_group")
+
+    function_logger.info(
+        f"START _create_first_group, with name '{FRACTAL_DEFAULT_GROUP_NAME}'"
     )
     with next(get_sync_db()) as db:
         group_all = db.execute(select(UserGroup))
         if group_all.scalars().one_or_none() is None:
             first_group = UserGroup(name=FRACTAL_DEFAULT_GROUP_NAME)
             db.add(first_group)
             db.commit()
-            logger.info(f"Created group {FRACTAL_DEFAULT_GROUP_NAME}")
+            function_logger.info(
+                f"Created group '{FRACTAL_DEFAULT_GROUP_NAME}'"
+            )
         else:
-            logger.info(
-                f"Group {FRACTAL_DEFAULT_GROUP_NAME} already exists, skip."
+            function_logger.info(
+                f"Group '{FRACTAL_DEFAULT_GROUP_NAME}' already exists, skip."
             )
-    logger.info(
-        f"END   _create_first_group, with name {FRACTAL_DEFAULT_GROUP_NAME}"
+    function_logger.info(
+        f"END   _create_first_group, with name '{FRACTAL_DEFAULT_GROUP_NAME}'"
     )

tests/data/testing_databases/.gitignore

@@ -1 +1,3 @@
 current_dump.sql
+venv*
+venv

tests/data/testing_databases/README.md

@@ -0,0 +1,38 @@
+Here is an example of how to add a new testing database, e.g. to create the `clean_db_fractal_2.4.0.sql` based on `clean_db_fractal_2.0.4.sql`.
+
+Note, as usual, that the `psql` and `createdb` may require additional options for authorization.
+
+
+1. Load existing database into postgres
+```bash
+DB_NAME="tmp_fractal_$(date +%s)"
+OLD_DB_FILE=./clean_db_fractal_2.0.4.sql
+NEW_FRACTAL_SERVER_VERSION=2.4.0
+
+echo "DB_NAME: $DB_NAME"
+
+dropdb "$DB_NAME"
+createdb "$DB_NAME"
+psql -d "$DB_NAME" -f "$OLD_DB_FILE"
+
+
+python3 -m venv venv
+source venv/bin/activate
+python3 -m pip install "fractal-server[postgres-psycopg-binary]==$NEW_FRACTAL_SERVER_VERSION"
+
+# Make sure that .fractal_server.env looks similar to
+echo "POSTGRES_DB=$DB_NAME" >> .fractal_server.env
+```
+
+2. Manually edit `.fractal_server.env`, so that there is a single `POSTGRES_DB`
+   line. Also make sure that `DB_ENGINE=postgres-psycopg`.
+
+3. Run migrations and dump db to file:
+```bash
+NEW_DB_FILE=./clean_db_fractal_2.4.0-tmp.sql
+
+fractalctl set-db
+fractalctl update-db-data  # and answer yes
+
+pg_dump --format=plain --file="$NEW_DB_FILE" $DB_NAME
+```