♻️ webserver: fixes mypy issues in products plugin (1 part) (#4256)

.codeclimate.yml

@@ -82,3 +82,4 @@ exclude_patterns:
   - "**/sandbox/"
   - packages/models-library/src/models_library/utils/_original_fastapi_encoders.py
   - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py
+  - services/web/server/src/simcore_service_webserver/projects/db.py  # NOTE: refactor will be done in upcomming PRs

api/specs/webserver/openapi-projects-ports.yaml

@@ -20,7 +20,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects.projects_ports_handlers.ProjectInputGet__'
+                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects._handlers__ports.ProjectInputGet__'
     patch:
       tags:
       - project
@@ -50,7 +50,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects.projects_ports_handlers.ProjectInputGet__'
+                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects._handlers__ports.ProjectInputGet__'
   /projects/{project_id}/outputs:
     get:
       tags:
@@ -72,7 +72,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects.projects_ports_handlers.ProjectOutputGet__'
+                $ref: '#/components/schemas/Envelope_dict_uuid.UUID__simcore_service_webserver.projects._handlers__ports.ProjectOutputGet__'
   /projects/{project_id}/metadata/ports:
     get:
       tags:
@@ -94,11 +94,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Envelope_list_simcore_service_webserver.projects.projects_ports_handlers.ProjectMetadataPortGet__'
+                $ref: '#/components/schemas/Envelope_list_simcore_service_webserver.projects._handlers__ports.ProjectMetadataPortGet__'
 components:
   schemas:
-    Envelope_dict_uuid.UUID__simcore_service_webserver.projects.projects_ports_handlers.ProjectInputGet__:
-      title: Envelope[dict[uuid.UUID, simcore_service_webserver.projects.projects_ports_handlers.ProjectInputGet]]
+    Envelope_dict_uuid.UUID__simcore_service_webserver.projects._handlers__ports.ProjectInputGet__:
+      title: Envelope[dict[uuid.UUID, simcore_service_webserver.projects._handlers__ports.ProjectInputGet]]
       type: object
       properties:
         data:
@@ -108,8 +108,8 @@ components:
             $ref: '#/components/schemas/ProjectInputGet'
         error:
           title: Error
-    Envelope_dict_uuid.UUID__simcore_service_webserver.projects.projects_ports_handlers.ProjectOutputGet__:
-      title: Envelope[dict[uuid.UUID, simcore_service_webserver.projects.projects_ports_handlers.ProjectOutputGet]]
+    Envelope_dict_uuid.UUID__simcore_service_webserver.projects._handlers__ports.ProjectOutputGet__:
+      title: Envelope[dict[uuid.UUID, simcore_service_webserver.projects._handlers__ports.ProjectOutputGet]]
       type: object
       properties:
         data:
@@ -119,8 +119,8 @@ components:
             $ref: '#/components/schemas/ProjectOutputGet'
         error:
           title: Error
-    Envelope_list_simcore_service_webserver.projects.projects_ports_handlers.ProjectMetadataPortGet__:
-      title: Envelope[list[simcore_service_webserver.projects.projects_ports_handlers.ProjectMetadataPortGet]]
+    Envelope_list_simcore_service_webserver.projects._handlers__ports.ProjectMetadataPortGet__:
+      title: Envelope[list[simcore_service_webserver.projects._handlers__ports.ProjectMetadataPortGet]]
       type: object
       properties:
         data:

api/specs/webserver/scripts/openapi_projects_crud.py

@@ -22,6 +22,13 @@
 from models_library.rest_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, Page
 from pydantic import NonNegativeInt
 from servicelib.aiohttp.long_running_tasks.server import TaskGet
+from simcore_service_webserver.projects._handlers_crud import (
+    ProjectPathParams,
+    ProjectTypeAPI,
+    _ProjectActiveParams,
+    _ProjectCreateParams,
+    _ProjectListParams,
+)
 from simcore_service_webserver.projects._rest_schemas import (
     ProjectCopyOverride,
     ProjectCreateNew,
@@ -31,13 +38,6 @@
     ProjectUpdate,
     TaskGet,
 )
-from simcore_service_webserver.projects.projects_handlers_crud import (
-    ProjectPathParams,
-    ProjectTypeAPI,
-    _ProjectActiveParams,
-    _ProjectCreateParams,
-    _ProjectListParams,
-)
 
 app = FastAPI(redoc_url=None)
 

api/specs/webserver/scripts/openapi_projects_ports.py

@@ -10,13 +10,12 @@
 
 
 from enum import Enum
-from typing import Union
 
 from fastapi import FastAPI
 from models_library.generics import Envelope
 from models_library.projects import ProjectID
 from models_library.projects_nodes import NodeID
-from simcore_service_webserver.projects.projects_ports_handlers import (
+from simcore_service_webserver.projects._handlers_project_ports import (
     ProjectInputGet,
     ProjectInputUpdate,
     ProjectMetadataPortGet,
@@ -25,7 +24,7 @@
 
 app = FastAPI(redoc_url=None)
 
-TAGS: list[Union[str, Enum]] = [
+TAGS: list[str | Enum] = [
     "project",
 ]
 

packages/pytest-simcore/src/pytest_simcore/helpers/faker_webserver.py

@@ -1,7 +1,7 @@
 from typing import Any, Final
 
 # Web-server API responses of /projects/{project_id}/metadata/ports
-# as reponses in this mock. SEE services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py
+# as reponses in this mock. SEE services/web/server/tests/unit/with_dbs/02/test__handlers__ports.py
 # NOTE: this could be added as examples in the OAS but for the moment we want to avoid overloading openapi.yml
 # in the web-server.
 PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA: Final[list[dict[str, Any]]] = [

packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects.py

@@ -6,16 +6,13 @@
 import json
 import uuid as uuidlib
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 
 from aiohttp import web
 from aiohttp.test_utils import TestClient
-from simcore_service_webserver.projects.project_models import ProjectDict
-from simcore_service_webserver.projects.projects_db import (
-    APP_PROJECT_DBAPI,
-    ProjectDBAPI,
-)
-from simcore_service_webserver.projects.projects_db_utils import DB_EXCLUSIVE_COLUMNS
+from simcore_service_webserver.projects._db_utils import DB_EXCLUSIVE_COLUMNS
+from simcore_service_webserver.projects.db import APP_PROJECT_DBAPI, ProjectDBAPI
+from simcore_service_webserver.projects.models import ProjectDict
 from simcore_service_webserver.utils import now_str
 
 from .utils_assert import assert_status
@@ -36,11 +33,11 @@ def empty_project_data():
 
 async def create_project(
     app: web.Application,
-    params_override: Optional[dict[str, Any]] = None,
-    user_id: Optional[int] = None,
+    params_override: dict[str, Any] | None = None,
+    user_id: int | None = None,
     *,
     product_name: str,
-    default_project_json: Optional[Path] = None,
+    default_project_json: Path | None = None,
     force_uuid: bool = False,
     as_template: bool = False,
 ) -> ProjectDict:
@@ -98,10 +95,10 @@ async def delete_all_projects(app: web.Application):
 class NewProject:
     def __init__(
         self,
-        params_override: Optional[dict] = None,
-        app: Optional[web.Application] = None,
+        params_override: dict | None = None,
+        app: web.Application | None = None,
         clear_all: bool = True,
-        user_id: Optional[int] = None,
+        user_id: int | None = None,
         *,
         product_name: str,
         tests_data_dir: Path,

scripts/demo/template-projects/converter.py

@@ -3,7 +3,7 @@
 import sys
 from pathlib import Path
 
-from simcore_service_webserver.projects.projects_db_utils import convert_to_schema_names
+from simcore_service_webserver.projects._db_utils import convert_to_schema_names
 
 SEPARATOR = ","
 

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -5439,7 +5439,7 @@ paths:
           content:
             application/json:
               schema:
-                title: 'Envelope[dict[uuid.UUID, simcore_service_webserver.projects.projects_ports_handlers.ProjectInputGet]]'
+                title: 'Envelope[dict[uuid.UUID, simcore_service_webserver.projects._handlers__ports.ProjectInputGet]]'
                 type: object
                 properties:
                   data:
@@ -5530,7 +5530,7 @@ paths:
           content:
             application/json:
               schema:
-                title: 'Envelope[dict[uuid.UUID, simcore_service_webserver.projects.projects_ports_handlers.ProjectOutputGet]]'
+                title: 'Envelope[dict[uuid.UUID, simcore_service_webserver.projects._handlers__ports.ProjectOutputGet]]'
                 type: object
                 properties:
                   data:
@@ -5578,7 +5578,7 @@ paths:
           content:
             application/json:
               schema:
-                title: 'Envelope[list[simcore_service_webserver.projects.projects_ports_handlers.ProjectMetadataPortGet]]'
+                title: 'Envelope[list[simcore_service_webserver.projects._handlers__ports.ProjectMetadataPortGet]]'
                 type: object
                 properties:
                   data:

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -35,7 +35,7 @@
 from .garbage_collector_settings import GarbageCollectorSettings
 from .invitations.settings import InvitationsSettings
 from .login.settings import LoginSettings
-from .projects.projects_settings import ProjectsSettings
+from .projects.settings import ProjectsSettings
 from .resource_manager.settings import ResourceManagerSettings
 from .rest_settings import RestSettings
 from .scicrunch.settings import SciCrunchSettings

services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py

@@ -39,9 +39,9 @@
 )
 
 from ...director_v2.api import create_or_update_pipeline
+from ...projects.db import APP_PROJECT_DBAPI, ProjectDBAPI
+from ...projects.exceptions import ProjectsException
 from ...projects.projects_api import get_project_for_user, submit_delete_project_task
-from ...projects.projects_db import APP_PROJECT_DBAPI, ProjectDBAPI
-from ...projects.projects_exceptions import ProjectsException
 from ...users.api import get_user
 from ...utils import now_str
 from ..exceptions import ExporterException

services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v2.py

@@ -10,8 +10,8 @@
 from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources
 
 from ...catalog.client import get_service
+from ...projects.exceptions import ProjectsException
 from ...projects.projects_api import get_project_for_user
-from ...projects.projects_exceptions import ProjectsException
 from ...scicrunch.db import ResearchResourceRepository
 from ..exceptions import ExporterException
 from .base_formatter import BaseFormatter

services/web/server/src/simcore_service_webserver/exporter/request_handlers.py

@@ -10,7 +10,7 @@
 
 from .._constants import RQ_PRODUCT_KEY
 from ..login.decorators import RQT_USERID_KEY, login_required
-from ..projects.project_lock import lock_project
+from ..projects.lock import lock_project
 from ..projects.projects_api import retrieve_and_notify_project_locked_state
 from ..security.decorators import permission_required
 from ..users.api import get_user_name

services/web/server/src/simcore_service_webserver/garbage_collector.py

@@ -8,7 +8,7 @@
 from .garbage_collector_tasks_api_keys import create_background_task_to_prune_api_keys
 from .garbage_collector_tasks_users import create_background_task_for_trial_accounts
 from .login.plugin import setup_login_storage
-from .projects.plugin import setup_projects_db
+from .projects.db import setup_projects_db
 from .socketio.plugin import setup_socketio_server
 
 logger = logging.getLogger(__name__)

services/web/server/src/simcore_service_webserver/garbage_collector_core.py

@@ -22,19 +22,19 @@
 from .director_v2.exceptions import ServiceWaitingForManualIntervention
 from .garbage_collector_settings import GUEST_USER_RC_LOCK_FORMAT
 from .garbage_collector_utils import get_new_project_owner_gid, replace_current_owner
+from .projects.db import ProjectDBAPI
+from .projects.exceptions import (
+    ProjectDeleteError,
+    ProjectLockError,
+    ProjectNotFoundError,
+)
 from .projects.projects_api import (
     get_project_for_user,
     get_workbench_node_ids_from_project_uuid,
     is_node_id_present_in_any_project_workbench,
     remove_project_dynamic_services,
     submit_delete_project_task,
 )
-from .projects.projects_db import ProjectDBAPI
-from .projects.projects_exceptions import (
-    ProjectDeleteError,
-    ProjectLockError,
-    ProjectNotFoundError,
-)
 from .redis import get_redis_lock_manager_client
 from .resource_manager.registry import RedisResourceRegistry, get_registry
 from .users import exceptions

services/web/server/src/simcore_service_webserver/garbage_collector_utils.py

@@ -8,8 +8,8 @@
 
 from .db_models import GroupType
 from .groups.api import get_group_from_gid
-from .projects.projects_db import APP_PROJECT_DBAPI, ProjectAccessRights
-from .projects.projects_exceptions import ProjectNotFoundError
+from .projects.db import APP_PROJECT_DBAPI, ProjectAccessRights
+from .projects.exceptions import ProjectNotFoundError
 from .users import exceptions
 from .users.api import get_user, get_user_id_from_gid, get_users_in_group
 from .users.exceptions import UserNotFoundError

services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py

@@ -19,7 +19,7 @@
 from pydantic.fields import Field
 from pydantic.types import PositiveInt
 
-from ..projects.project_models import ProjectDict
+from ..projects.models import ProjectDict
 from ..utils import compute_sha1_on_small_dataset, now_str
 from ..version_control.errors import UserUndefined
 from ..version_control.models import CommitID

services/web/server/src/simcore_service_webserver/meta_modeling/_projects.py

@@ -16,7 +16,7 @@
 
 from .._meta import api_version_prefix as VTAG
 from ..director_v2.api import AbstractProjectRunPolicy
-from ..projects.projects_handlers_crud import RQ_REQUESTED_REPO_PROJECT_UUID_KEY
+from ..projects._handlers_crud import RQ_REQUESTED_REPO_PROJECT_UUID_KEY
 from ..version_control.models import CommitID
 from ._iterations import get_or_create_runnable_projects, get_runnable_projects_ids
 from ._version_control import VersionControlForMetaModeling

services/web/server/src/simcore_service_webserver/meta_modeling/_version_control.py

@@ -9,7 +9,7 @@
 from models_library.projects import ProjectIDStr
 from models_library.utils.fastapi_encoders import jsonable_encoder
 
-from ..projects.project_models import ProjectDict
+from ..projects.models import ProjectDict
 from ..version_control.db import VersionControlRepository
 from ..version_control.errors import UserUndefined
 from ..version_control.models import CommitID, TagProxy

services/web/server/src/simcore_service_webserver/notifications/_db_comp_tasks_listening_task.py

@@ -21,8 +21,8 @@
 from simcore_postgres_database.webserver_models import DB_CHANNEL_NAME, projects
 from sqlalchemy.sql import select
 
-from ..projects import projects_api, projects_exceptions
-from ..projects.projects_nodes_utils import update_node_outputs
+from ..projects import exceptions, projects_api
+from ..projects.nodes_utils import update_node_outputs
 from ._utils import convert_state_from_db
 
 _logger = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ async def _get_project_owner(conn: SAConnection, project_uuid: str) -> PositiveI
         select([projects.c.prj_owner]).where(projects.c.uuid == project_uuid)
     )
     if not the_project_owner:
-        raise projects_exceptions.ProjectOwnerNotFoundError(project_uuid)
+        raise exceptions.ProjectOwnerNotFoundError(project_uuid)
     return the_project_owner
 
 
@@ -110,17 +110,17 @@ async def _handle_db_notification(
                 node_errors=task_data.get("errors", None),
             )
 
-    except projects_exceptions.ProjectNotFoundError as exc:
+    except exceptions.ProjectNotFoundError as exc:
         _logger.warning(
             "Project %s was not found and cannot be updated. Maybe was it deleted?",
             exc.project_uuid,
         )
-    except projects_exceptions.ProjectOwnerNotFoundError as exc:
+    except exceptions.ProjectOwnerNotFoundError as exc:
         _logger.warning(
             "Project owner of project %s could not be found, is the project valid?",
             exc.project_uuid,
         )
-    except projects_exceptions.NodeNotFoundError as exc:
+    except exceptions.NodeNotFoundError as exc:
         _logger.warning(
             "Node %s of project %s not found and cannot be updated. Maybe was it deleted?",
             exc.node_uuid,

services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_consumers.py

@@ -24,7 +24,7 @@
 from servicelib.utils import logged_gather
 
 from ..projects import projects_api
-from ..projects.projects_exceptions import NodeNotFoundError, ProjectNotFoundError
+from ..projects.exceptions import NodeNotFoundError, ProjectNotFoundError
 from ..rabbitmq import get_rabbitmq_client
 from ..socketio.messages import (
     SOCKET_IO_EVENT,

services/web/server/src/simcore_service_webserver/projects/_create_utils.py → services/web/server/src/simcore_service_webserver/projects/_crud_create_utils.py

@@ -25,10 +25,10 @@
 from . import projects_api
 from ._permalink import update_or_pop_permalink_in_project
 from ._rest_schemas import ProjectGet
-from .project_models import ProjectDict
-from .projects_db import ProjectDBAPI
-from .projects_exceptions import ProjectInvalidRightsError, ProjectNotFoundError
-from .projects_utils import NodesMap, clone_project_document, default_copy_project_name
+from .db import ProjectDBAPI
+from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError
+from .models import ProjectDict
+from .utils import NodesMap, clone_project_document, default_copy_project_name
 
 OVERRIDABLE_DOCUMENT_KEYS = [
     "name",