🐛 Project copy failing when pennsieve token is active (#3509)

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -489,7 +489,7 @@ async def deep_copy_project_simcore_s3(
                     )
                     for output in node.get("outputs", {}).values()
                     if isinstance(output, dict)
-                    and int(output.get("store", self.location_id)) == DATCORE_ID
+                    and (int(output.get("store", self.location_id)) == DATCORE_ID)
                 ]
             )
         await logged_gather(*copy_tasks, max_concurrency=MAX_CONCURRENT_S3_TASKS)

services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py

@@ -40,7 +40,10 @@
 from ..rest_constants import RESPONSE_MODEL_POLICY
 from ..security_api import check_permission
 from ..security_decorators import permission_required
-from ..storage_api import copy_data_folders_from_project, get_project_total_size
+from ..storage_api import (
+    copy_data_folders_from_project,
+    get_project_total_size_simcore_s3,
+)
 from ..users_api import get_user_name
 from . import projects_api
 from .project_models import ProjectDict, ProjectTypeAPI
@@ -164,7 +167,9 @@ async def _prepare_project_copy(
     assert settings  # nosec
     if max_bytes := settings.PROJECTS_MAX_COPY_SIZE_BYTES:
         # get project total data size
-        project_data_size = await get_project_total_size(app, user_id, src_project_uuid)
+        project_data_size = await get_project_total_size_simcore_s3(
+            app, user_id, src_project_uuid
+        )
         if project_data_size >= max_bytes:
             raise web.HTTPUnprocessableEntity(
                 reason=f"Source project data size is {project_data_size.human_readable()}."

services/web/server/src/simcore_service_webserver/storage_api.py

@@ -6,7 +6,11 @@
 from typing import Any, AsyncGenerator
 
 from aiohttp import ClientError, ClientSession, ClientTimeout, web
-from models_library.api_schemas_storage import FileLocationArray, FileMetaDataGet
+from models_library.api_schemas_storage import (
+    FileLocation,
+    FileLocationArray,
+    FileMetaDataGet,
+)
 from models_library.generics import Envelope
 from models_library.projects import ProjectID
 from models_library.users import UserID
@@ -18,6 +22,7 @@
     LRTask,
     long_running_task_request,
 )
+from servicelib.logging_utils import log_context
 from simcore_service_webserver.projects.project_models import ProjectDict
 from simcore_service_webserver.projects.projects_utils import NodesMap
 from yarl import URL
@@ -54,31 +59,34 @@ async def get_storage_locations(
         return locations_enveloped.data
 
 
-async def get_project_total_size(
+async def get_project_total_size_simcore_s3(
     app: web.Application, user_id: UserID, project_uuid: ProjectID
 ) -> ByteSize:
-    log.debug("getting %s total size for %s", f"{project_uuid=}", f"{user_id=}")
-    user_accessible_locations = await get_storage_locations(app, user_id)
-    session, api_endpoint = _get_storage_client(app)
-
-    project_size_bytes = 0
-    for location in user_accessible_locations:
-        files_metadata_url = (
-            api_endpoint / "locations" / f"{location.id}" / "files" / "metadata"
-        ).with_query(user_id=user_id, uuid_filter=f"{project_uuid}")
-        async with session.get(f"{files_metadata_url}") as response:
-            response.raise_for_status()
-            list_of_files_enveloped = Envelope[list[FileMetaDataGet]].parse_obj(
-                await response.json()
-            )
-            assert list_of_files_enveloped.data is not None  # nosec
-        for file_metadata in list_of_files_enveloped.data:
-            project_size_bytes += file_metadata.file_size
-    project_size = parse_obj_as(ByteSize, project_size_bytes)
-    log.info(
-        "%s total size is %s", f"{project_uuid}", f"{project_size.human_readable()}"
-    )
-    return project_size
+    with log_context(
+        log,
+        logging.DEBUG,
+        msg=f"getting {project_uuid=} total size in S3 for {user_id=}",
+    ):
+        # NOTE: datcore does not handle filtering and is too slow for this, so for now this is hard-coded
+        user_accessible_locations = [FileLocation(name="simcore.s3", id=0)]
+        session, api_endpoint = _get_storage_client(app)
+
+        project_size_bytes = 0
+        for location in user_accessible_locations:
+            files_metadata_url = (
+                api_endpoint / "locations" / f"{location.id}" / "files" / "metadata"
+            ).with_query(user_id=user_id, uuid_filter=f"{project_uuid}")
+            async with session.get(f"{files_metadata_url}") as response:
+                response.raise_for_status()
+                list_of_files_enveloped = Envelope[list[FileMetaDataGet]].parse_obj(
+                    await response.json()
+                )
+                assert list_of_files_enveloped.data is not None  # nosec
+            for file_metadata in list_of_files_enveloped.data:
+                project_size_bytes += file_metadata.file_size
+        project_size = parse_obj_as(ByteSize, project_size_bytes)
+
+        return project_size
 
 
 async def copy_data_folders_from_project(

services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py

@@ -271,8 +271,8 @@ async def test_copying_too_large_project_returns_422(
     large_project_total_size = (
         app_settings.WEBSERVER_PROJECTS.PROJECTS_MAX_COPY_SIZE_BYTES + 1
     )
-    storage_subsystem_mock.get_project_total_size.return_value = parse_obj_as(
-        ByteSize, large_project_total_size
+    storage_subsystem_mock.get_project_total_size_simcore_s3.return_value = (
+        parse_obj_as(ByteSize, large_project_total_size)
     )
 
     # POST /v0/projects

services/web/server/tests/unit/with_dbs/_helpers.py

@@ -122,4 +122,4 @@ class MockedStorageSubsystem(NamedTuple):
     copy_data_folders_from_project: mock.MagicMock
     delete_project: mock.MagicMock
     delete_node: mock.MagicMock
-    get_project_total_size: mock.MagicMock
+    get_project_total_size_simcore_s3: mock.MagicMock

services/web/server/tests/unit/with_dbs/conftest.py

@@ -275,7 +275,7 @@ async def _mock_result():
     )
 
     mock3 = mocker.patch(
-        "simcore_service_webserver.projects.projects_handlers_crud.get_project_total_size",
+        "simcore_service_webserver.projects.projects_handlers_crud.get_project_total_size_simcore_s3",
         autospec=True,
         return_value=parse_obj_as(ByteSize, "1Gib"),
     )