Introduce user settings

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: ci
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
   pull_request:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
 
 jobs:
 

.github/workflows/migrations.yml

@@ -9,9 +9,9 @@ name: migrations
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
   pull_request:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
 
 
 env:

.github/workflows/precommit.yml

@@ -2,9 +2,9 @@ name: precommit
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
   pull_request:
-    branches: ["main"]
+    branches: ["main", "1774-introduce-user-settings"]
 
 jobs:
   precommit:

CHANGELOG.md

@@ -1,5 +1,19 @@
 **Note**: Numbers like (\#1234) point to closed Pull Requests on the fractal-server repository.
 
+# 2.6.0 (unreleased)
+
+> WARNING: This release requires running `fractalctl update-db-data` (after `fractalctl set-db`).
+
+* API:
+    * Introduce user-settings API, in `/auth/users/{user_id}/settings/` and `/auth/current-user/settings/` (\#1778).
+    * Add the creation of empty settings to `UserManager.on_after_register` hook (\#1778).
+    * Remove deprecated user's attributes (`slurm_user`, `cache_dir`, `slurm_accounts`) from API, in favor of new `UserSetting` ones (\#1778).
+    * Validate user settings in endpoints that rely on them (\#1778).
+    * Propagate user settings to background tasks when needed (\#1778).
+* Database:
+    * Introduce new `user_settings` table, and link it to `user_oauth` (\#1778).
+
+
 # 2.5.2
 
 * App:

docs/development.md

@@ -72,7 +72,7 @@ an incremental migration script, as in
 ```
 $ export SQLITE_PATH=some-test.db
 $ rm some-test.db
-$ poetry run fractalctl set-db
+$ poetry run fractalctl set-db --skip-init-data
 $ poetry run alembic revision --autogenerate -m "Some migration message"
 
 # UserWarning: SQLite is partially supported but discouraged in production environment.SQLite offers partial support for ForeignKey constraints. As such, consistency of the database cannot be guaranteed.

fractal_server/__main__.py

@@ -43,9 +43,17 @@
 )
 
 # fractalctl set-db
-subparsers.add_parser(
+set_db_parser = subparsers.add_parser(
     "set-db",
-    description="Initialise the database and apply schema migrations",
+    description=(
+        "Initialise/upgrade database schemas and create first group&user."
+    ),
+)
+set_db_parser.add_argument(
+    "--skip-init-data",
+    action="store_true",
+    help="If set, do not try creating first group and user.",
+    default=False,
 )
 
 # fractalctl update-db-data
@@ -66,27 +74,34 @@ def save_openapi(dest="openapi.json"):
         json.dump(openapi_schema, f)
 
 
-def set_db():
+def set_db(skip_init_data: bool = False):
     """
-    Set-up / Upgrade database schema
+    Upgrade database schema *and* create first group/user
 
     Call alembic to upgrade to the latest migration.
-
     Ref: https://stackoverflow.com/a/56683030/283972
+
+    Arguments:
+        skip_init_data: If `True`, skip creation of first group and user.
     """
-    import alembic.config
-    from pathlib import Path
-    import fractal_server
     from fractal_server.app.security import _create_first_user
     from fractal_server.app.security import _create_first_group
     from fractal_server.syringe import Inject
     from fractal_server.config import get_settings
 
+    import alembic.config
+    from pathlib import Path
+    import fractal_server
+
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
     alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
     print(f"START: Run alembic.config, with argv={alembic_args}")
     alembic.config.main(argv=alembic_args)
     print("END: alembic.config")
+
+    if skip_init_data:
+        return
+
     # Insert default group
     print()
     _create_first_group()
@@ -179,7 +194,7 @@ def run():
     if args.cmd == "openapi":
         save_openapi(dest=args.openapi_file)
     elif args.cmd == "set-db":
-        set_db()
+        set_db(skip_init_data=args.skip_init_data)
     elif args.cmd == "update-db-data":
         update_db_data()
     elif args.cmd == "start":

fractal_server/app/models/__init__.py

@@ -7,5 +7,6 @@
 from .linkuserproject import LinkUserProject  # noqa: F401
 from .linkuserproject import LinkUserProjectV2  # noqa: F401
 from .security import *  # noqa
+from .user_settings import UserSettings  # noqa
 from .v1 import *  # noqa
 from .v2 import *  # noqa

fractal_server/app/models/security.py

@@ -20,6 +20,7 @@
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
+from .user_settings import UserSettings
 from fractal_server.utils import get_timestamp
 
 
@@ -104,6 +105,13 @@ class UserOAuth(SQLModel, table=True):
         sa_relationship_kwargs={"lazy": "joined", "cascade": "all, delete"},
     )
 
+    user_settings_id: Optional[int] = Field(
+        foreign_key="user_settings.id", default=None
+    )
+    settings: Optional[UserSettings] = Relationship(
+        sa_relationship_kwargs=dict(lazy="selectin", cascade="all, delete")
+    )
+
     class Config:
         orm_mode = True
 

fractal_server/app/models/user_settings.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+
+class UserSettings(SQLModel, table=True):
+    """
+    Comprehensive list of user settings.
+
+    Attributes:
+        id: ID of database object
+        slurm_accounts:
+            List of SLURM accounts, to be used upon Fractal job submission.
+        ssh_host: SSH-reachable host where a SLURM client is available.
+        ssh_username: User on `ssh_host`.
+        ssh_private_key_path: Path of private SSH key for `ssh_username`.
+        ssh_tasks_dir: Task-venvs base folder on `ssh_host`.
+        ssh_jobs_dir: Jobs base folder on `ssh_host`.
+        slurm_user: Local user, to be impersonated via `sudo -u`
+        cache_dir: Folder where `slurm_user` can write.
+    """
+
+    __tablename__ = "user_settings"
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    slurm_accounts: list[str] = Field(
+        sa_column=Column(JSON, server_default="[]", nullable=False)
+    )
+    ssh_host: Optional[str] = None
+    ssh_username: Optional[str] = None
+    ssh_private_key_path: Optional[str] = None
+    ssh_tasks_dir: Optional[str] = None
+    ssh_jobs_dir: Optional[str] = None
+    slurm_user: Optional[str] = None
+    cache_dir: Optional[str] = None

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -22,6 +22,7 @@
 from ....models.v1 import Workflow
 from ....models.v1 import WorkflowTask
 from ....schemas.v1 import JobStatusTypeV1
+from ...aux.validate_user_settings import verify_user_has_settings
 from fractal_server.app.models import UserOAuth
 
 
@@ -367,7 +368,11 @@ async def _get_task_check_owner(
                 ),
             )
         else:
-            owner = user.username or user.slurm_user
+            if user.username:
+                owner = user.username
+            else:
+                verify_user_has_settings(user)
+                owner = user.settings.slurm_user
             if owner != task.owner:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,

fractal_server/app/routes/api/v1/project.py

@@ -34,6 +34,7 @@
 from ....schemas.v1 import ProjectCreateV1
 from ....schemas.v1 import ProjectReadV1
 from ....schemas.v1 import ProjectUpdateV1
+from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _check_project_exists
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
@@ -321,25 +322,11 @@ async def apply_workflow(
             ),
         )
 
-    # If backend is SLURM, check that the user has required attributes
-    backend = settings.FRACTAL_RUNNER_BACKEND
-    if backend == "slurm":
-        if not user.slurm_user:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    f"FRACTAL_RUNNER_BACKEND={backend}, "
-                    f"but {user.slurm_user=}."
-                ),
-            )
-        if not user.cache_dir:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    f"FRACTAL_RUNNER_BACKEND={backend}, "
-                    f"but {user.cache_dir=}."
-                ),
-            )
+    # Validate user settings
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    user_settings = await validate_user_settings(
+        user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
+    )
 
     # Check that datasets have the right number of resources
     if not input_dataset.resource_list:
@@ -386,7 +373,7 @@ async def apply_workflow(
         )
 
     if apply_workflow.slurm_account is not None:
-        if apply_workflow.slurm_account not in user.slurm_accounts:
+        if apply_workflow.slurm_account not in user_settings.slurm_accounts:
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail=(
@@ -395,8 +382,8 @@ async def apply_workflow(
                 ),
             )
     else:
-        if len(user.slurm_accounts) > 0:
-            apply_workflow.slurm_account = user.slurm_accounts[0]
+        if len(user_settings.slurm_accounts) > 0:
+            apply_workflow.slurm_account = user_settings.slurm_accounts[0]
 
     # Add new ApplyWorkflow object to DB
     job = ApplyWorkflow(
@@ -480,8 +467,8 @@ async def apply_workflow(
         output_dataset_id=output_dataset.id,
         job_id=job.id,
         worker_init=apply_workflow.worker_init,
-        slurm_user=user.slurm_user,
-        user_cache_dir=user.cache_dir,
+        slurm_user=user_settings.slurm_user,
+        user_cache_dir=user_settings.cache_dir,
     )
     request.app.state.jobsV1.append(job.id)
     logger.info(

fractal_server/app/routes/api/v1/task.py

@@ -17,6 +17,7 @@
 from ....schemas.v1 import TaskCreateV1
 from ....schemas.v1 import TaskReadV1
 from ....schemas.v1 import TaskUpdateV1
+from ...aux.validate_user_settings import verify_user_has_settings
 from ._aux_functions import _get_task_check_owner
 from ._aux_functions import _raise_if_v1_is_read_only
 from fractal_server.app.models import UserOAuth
@@ -126,16 +127,18 @@ async def create_task(
     # Set task.owner attribute
     if user.username:
         owner = user.username
-    elif user.slurm_user:
-        owner = user.slurm_user
     else:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                "Cannot add a new task because current user does not "
-                "have `username` or `slurm_user` attributes."
-            ),
-        )
+        verify_user_has_settings(user)
+        if user.settings.slurm_user:
+            owner = user.settings.slurm_user
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot add a new task because current user does not "
+                    "have `username` or `slurm_user` attributes."
+                ),
+            )
 
     # Prepend owner to task.source
     task.source = f"{owner}:{task.source}"

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,6 +21,7 @@
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
+from ...aux.validate_user_settings import verify_user_has_settings
 from fractal_server.app.models import UserOAuth
 from fractal_server.images import Filters
 
@@ -362,7 +363,11 @@ async def _get_task_check_owner(
                 ),
             )
         else:
-            owner = user.username or user.slurm_user
+            if user.username:
+                owner = user.username
+            else:
+                verify_user_has_settings(user)
+                owner = user.settings.slurm_user
             if owner != task.owner:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,