diff --git a/Makefile b/Makefile index ca530b619b27..449a0d291c6e 100644 --- a/Makefile +++ b/Makefile @@ -232,14 +232,15 @@ TableWidth=140;\ printf "%24s | %90s | %12s | %12s\n" Name Endpoint User Password;\ printf "%.$${TableWidth}s\n" "$$separator";\ printf "$$rows" 'oSparc platform' 'http://$(get_my_ip).nip.io:9081';\ -printf "$$rows" 'oSparc platform web API' 'http://$(get_my_ip).nip.io:9081/dev/doc';\ +printf "$$rows" 'oSparc web API doc' 'http://$(get_my_ip).nip.io:9081/dev/doc';\ +printf "$$rows" 'oSparc public API doc' 'http://$(get_my_ip).nip.io:8006/dev/doc';\ printf "$$rows" 'Postgres DB' 'http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\ printf "$$rows" Portainer 'http://$(get_my_ip).nip.io:9000' admin adminadmin;\ printf "$$rows" Redis 'http://$(get_my_ip).nip.io:18081';\ -printf "$$rows" 'Docker Registry' $${REGISTRY_URL} $${REGISTRY_USER} $${REGISTRY_PW};\ -printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787"; -printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/"; -printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15762" admin adminadmin; +printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\ +printf "$$rows" "Docker Registry" "$${REGISTRY_URL}" $${REGISTRY_USER} $${REGISTRY_PW};\ +printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15762" admin adminadmin;\ +printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\ printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\ echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081"; endef diff --git a/api/specs/webserver/openapi-meta-projects.yaml b/api/specs/webserver/openapi-meta-projects.yaml index a723501cfa09..ab3adcc5fe73 100644 --- a/api/specs/webserver/openapi-meta-projects.yaml +++ b/api/specs/webserver/openapi-meta-projects.yaml @@ -52,7 +52,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Page_IterationAsItem_" + $ref: "#/components/schemas/Page_IterationItem_" "404": description: This project has no iterations.Only meta-project have iterations @@ -63,6 +63,141 @@ paths: application/json: schema: $ref: "#/components/schemas/HTTPValidationError" + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}: + get: + tags: + - meta-projects + summary: Get Project Iterations + description: Get current project's iterations + operationId: "simcore_service_webserver.meta_modeling_handlers._get_meta_project_iterations_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + name: iter_id + schema: + type: integer + in: path + responses: + "200": + description: Successful Response + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: + get: + tags: + - meta-projects + summary: List Project Iterations Results + description: Lists current project's iterations results table + operationId: "simcore_service_webserver.meta_modeling_handlers._list_meta_project_iterations_results_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + exclusiveMinimum: false + type: integer + description: index to the first item to return (pagination) + default: 0 + minimum: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50.0 + minimum: 1.0 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + "200": + description: Successful Response + content: + application/json: + schema: + $ref: "#/components/schemas/Page_IterationResultItem_" + "404": + description: + This project has no iterations.Only meta-project have iterations + and they must be explicitly created. + "422": + description: Validation Error + content: + application/json: + schema: + $ref: "#/components/schemas/HTTPValidationError" + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results: + get: + tags: + - meta-projects + summary: Get Project Iteration Results + description: Get current project's iterations + operationId: "simcore_service_webserver.meta_modeling_handlers._get_meta_project_iteration_results_handler" + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + schema: + type: integer + name: iter_id + in: path + + responses: + "200": + description: Successful Response + components: schemas: HTTPValidationError: @@ -74,8 +209,8 @@ components: type: array items: $ref: "#/components/schemas/ValidationError" - IterationAsItem: - title: IterationAsItem + IterationItem: + title: IterationItem required: - name - parent @@ -177,8 +312,8 @@ components: title: Count minimum: 0.0 type: integer - Page_IterationAsItem_: - title: Page[IterationAsItem] + Page_IterationItem_: + title: Page[IterationItem] required: - _meta - _links @@ -193,7 +328,7 @@ components: title: Data type: array items: - $ref: "#/components/schemas/IterationAsItem" + $ref: "#/components/schemas/IterationItem" ParentMetaProjectRef: title: ParentMetaProjectRef required: @@ -227,3 +362,21 @@ components: type: title: Error Type type: string + Page_IterationResultItem_: + title: Page[IterationResultItem] + required: + - _meta + - _links + - data + type: object + properties: + _meta: + $ref: "#/components/schemas/PageMetaInfoLimitOffset" + _links: + $ref: "#/components/schemas/PageLinks" + data: + title: Data + type: array + items: + $ref: "#/components/schemas/IterationItem" + # NOTE: intentionally wrong. Will be deprecated diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 6b1a5b971191..931a60feab05 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -223,6 +223,15 @@ paths: /projects/{project_uuid}/checkpoint/{ref_id}/iterations: $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations" + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1{iter_id}" + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1-~1results" + + /projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results: + $ref: "./openapi-meta-projects.yaml#/paths/~1projects~1{project_uuid}~1checkpoint~1{ref_id}~1iterations~1{iter_id}~1results" + # REPOSITORY ------------------------------------------------------------------------- /repos/projects: $ref: "./openapi-version-control.yaml#/paths/~1repos~1projects" diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index 045061e4d0fd..dca011007f9d 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -58,12 +58,18 @@ services: webserver: environment: - SC_BOOT_MODE=${SC_BOOT_MODE:-default} + - REST_SWAGGER_API_DOC_ENABLED=1 ports: - "8080" - "3001:3000" deploy: labels: - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=false + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.service=${SWARM_STACK_NAME}_webserver + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.rule=hostregexp(`{host:.+}`) && PathPrefix(`/dev/`) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.priority=3 + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry dask-sidecar: environment: @@ -134,8 +140,7 @@ services: - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - traefik.enable=true - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.service=api@internal - - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.rule=PathPrefix(`/dashboard`) - || PathPrefix(`/api`) + - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.rule=PathPrefix(`/dashboard`) || PathPrefix(`/api`) - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.entrypoints=traefik_monitor - traefik.http.routers.${SWARM_STACK_NAME}_api_internal.middlewares=${SWARM_STACK_NAME}_gzip@docker - traefik.http.services.${SWARM_STACK_NAME}_api_internal.loadbalancer.server.port=8080 diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d6334bfde49e..3c0677167285 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2943,7 +2943,7 @@ paths: content: application/json: schema: - title: 'Page[IterationAsItem]' + title: 'Page[IterationItem]' required: - _meta - _links @@ -3019,7 +3019,7 @@ paths: title: Data type: array items: - title: IterationAsItem + title: IterationItem required: - name - parent @@ -3098,6 +3098,149 @@ paths: type: title: Error Type type: string + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}': + get: + tags: + - meta-projects + summary: Get Project Iterations + description: Get current project's iterations + operationId: simcore_service_webserver.meta_modeling_handlers._get_meta_project_iterations_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + name: iter_id + schema: + type: integer + in: path + responses: + '200': + description: Successful Response + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results': + get: + tags: + - meta-projects + summary: List Project Iterations Results + description: Lists current project's iterations results table + operationId: simcore_service_webserver.meta_modeling_handlers._list_meta_project_iterations_results_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + exclusiveMinimum: false + type: integer + description: index to the first item to return (pagination) + default: 0 + minimum: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: 'Page[IterationResultItem]' + required: + - _meta + - _links + - data + type: object + properties: + _meta: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/_meta' + _links: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/_links' + data: + title: Data + type: array + items: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/200/content/application~1json/schema/properties/data/items' + '404': + description: This project has no iterations.Only meta-project have iterations and they must be explicitly created. + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/paths/~1projects~1%7Bproject_uuid%7D~1checkpoint~1%7Bref_id%7D~1iterations/get/responses/422/content/application~1json/schema' + '/projects/{project_uuid}/checkpoint/{ref_id}/iterations/{iter_id}/results': + get: + tags: + - meta-projects + summary: Get Project Iteration Results + description: Get current project's iterations + operationId: simcore_service_webserver.meta_modeling_handlers._get_meta_project_iteration_results_handler + parameters: + - description: Project unique identifier + required: true + schema: + title: Project Uuid + type: string + description: Project unique identifier + format: uuid + name: project_uuid + in: path + - required: true + schema: + title: Ref Id + anyOf: + - type: integer + - type: string + name: ref_id + in: path + - required: true + schema: + type: integer + name: iter_id + in: path + responses: + '200': + description: Successful Response /repos/projects: get: tags: diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py index 694ec652463a..fcd93e1ecd31 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_handlers.py @@ -2,18 +2,20 @@ """ import logging -from typing import List, NamedTuple, Optional, Tuple +from typing import Callable, List, NamedTuple, Optional from aiohttp import web from models_library.projects import ProjectID from models_library.rest_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, Page from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError, validator from pydantic.fields import Field from pydantic.networks import HttpUrl from ._meta import api_version_prefix as VTAG -from .meta_modeling_iterations import ProjectIteration +from .login.decorators import login_required +from .meta_modeling_iterations import IterationID, ProjectIteration +from .meta_modeling_results import ExtractedResults, extract_project_results from .meta_modeling_version_control import VersionControlForMetaModeling from .rest_constants import RESPONSE_MODEL_POLICY from .security_decorators import permission_required @@ -23,10 +25,33 @@ log = logging.getLogger(__name__) - # HANDLER'S CORE IMPLEMENTATION ------------------------------------------------------------ -IterationTuple = Tuple[ProjectID, CommitID] + +class _QueryParametersModel(BaseModel): + project_uuid: ProjectID + ref_id: CommitID + limit: int = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + offset: int = 0 + + @validator("ref_id", pre=True) + @classmethod + def tags_as_refid_not_implemented(cls, v): + try: + return CommitID(v) + except ValueError as err: + # e.g. HEAD + raise NotImplementedError( + "cannot convert ref (e.g. HEAD) -> commit id" + ) from err + + +def parse_query_parameters(request: web.Request) -> _QueryParametersModel: + try: + return _QueryParametersModel(**request.match_info) + except ValidationError as err: + # TODO: compose reason message better + raise web.HTTPUnprocessableEntity(reason=f"Invalid query parameters: {err}") class _NotTaggedAsIteration(Exception): @@ -37,8 +62,14 @@ class _NotTaggedAsIteration(Exception): ... +class IterationItem(NamedTuple): + project_id: ProjectID + commit_id: CommitID + iteration_index: IterationID + + class _IterationsRange(NamedTuple): - items: List[IterationTuple] + items: List[IterationItem] total_count: int @@ -64,7 +95,7 @@ async def _get_project_iterations_range( repo_id, commit_id ) - iterations: List[Tuple[ProjectID, CommitID]] = [] + iter_items: List[IterationItem] = [] for n, tags in enumerate(tags_per_child): try: iteration: Optional[ProjectIteration] = None @@ -93,7 +124,13 @@ async def _get_project_iterations_range( if not iteration: raise _NotTaggedAsIteration(f"No iteration tag found in {tags=}") - iterations.append((workcopy_id, iteration.iter_index)) + iter_items.append( + IterationItem( + project_id=workcopy_id, + commit_id=iteration.repo_commit_id, + iteration_index=iteration.iteration_index, + ) + ) except _NotTaggedAsIteration as err: log.warning( @@ -105,18 +142,19 @@ async def _get_project_iterations_range( ) # Selects range on those tagged as iterations and returned their assigned workcopy id - total_number_of_iterations = len(iterations) + total_number_of_iterations = len(iter_items) # sort and select. If requested interval is outside of range, it returns empty - iterations.sort(key=lambda tup: tup[1]) + iter_items.sort(key=lambda item: item.iteration_index) if limit is None: return _IterationsRange( - items=iterations[offset:], total_count=total_number_of_iterations + items=iter_items[offset:], + total_count=total_number_of_iterations, ) return _IterationsRange( - items=iterations[offset : (offset + limit)], + items=iter_items[offset : (offset + limit)], total_count=total_number_of_iterations, ) @@ -125,7 +163,7 @@ async def create_or_get_project_iterations( vc_repo: VersionControlForMetaModeling, project_uuid: ProjectID, commit_id: CommitID, -) -> List[IterationTuple]: +) -> List[IterationItem]: raise NotImplementedError() @@ -138,15 +176,25 @@ class ParentMetaProjectRef(BaseModel): ref_id: CheckpointID -class ProjectIterationAsItem(BaseModel): +class _BaseModelGet(BaseModel): name: str = Field( ..., - description="Iteration's resource name [AIP-122](https://google.aip.dev/122)", + description="Iteration's resource API name", + # TODO: PC x_mark_resouce_name=True, # [AIP-122](https://google.aip.dev/122) ) parent: ParentMetaProjectRef = Field( ..., description="Reference to the the meta-project that created this iteration" ) + url: HttpUrl = Field(..., description="self reference") + + +class ProjectIterationItem(_BaseModelGet): + iteration_index: IterationID = Field( + ..., + # TODO: PC x_mark_resource_id_segment=True, # [AIP-122](https://google.aip.dev/122) + ) + workcopy_project_id: ProjectID = Field( ..., description="ID to this iteration's working copy." @@ -156,11 +204,71 @@ class ProjectIterationAsItem(BaseModel): workcopy_project_url: HttpUrl = Field( ..., description="reference to a working copy project" ) - url: HttpUrl = Field(..., description="self reference") + + @classmethod + def create( + cls, + meta_project_uuid, + meta_project_commit_id, + iteration_index, + project_id, + url_for: Callable, + ): + return cls( + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{iteration_index}", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=iteration_index, + workcopy_project_id=project_id, + workcopy_project_url=url_for( + "get_project", + project_id=project_id, + ), + url=url_for( + f"{__name__}._get_meta_project_iterations_handler", + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, + iter_id=iteration_index, + ), + ) -# ROUTES ------------------------------------------------------------ +class ProjectIterationResultItem(ProjectIterationItem): + results: ExtractedResults + + @classmethod + def create( # pylint: disable=arguments-differ + cls, + meta_project_uuid, + meta_project_commit_id, + iteration_index, + project_id, + results, + url_for: Callable, + ): + return cls( + name=f"projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}/iterations/{iteration_index}/results", + parent=ParentMetaProjectRef( + project_id=meta_project_uuid, ref_id=meta_project_commit_id + ), + iteration_index=iteration_index, + workcopy_project_id=project_id, + results=results, + workcopy_project_url=url_for( + "get_project", + project_id=project_id, + ), + url=url_for( + f"{__name__}._get_meta_project_iteration_results_handler", + project_uuid=meta_project_uuid, + ref_id=meta_project_commit_id, + iter_id=iteration_index, + ), + ) + +# ROUTES ------------------------------------------------------------ routes = web.RouteTableDef() @@ -169,67 +277,55 @@ class ProjectIterationAsItem(BaseModel): f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations", name=f"{__name__}._list_meta_project_iterations_handler", ) +@login_required @permission_required("project.snapshot.read") async def _list_meta_project_iterations_handler(request: web.Request) -> web.Response: # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 # parse and validate request ---- + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - _project_uuid = ProjectID(request.match_info["project_uuid"]) - _ref_id = request.match_info["ref_id"] - - _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) - _offset = int(request.query.get("offset", 0)) - - try: - commit_id = CommitID(_ref_id) - except ValueError as err: - # e.g. HEAD - raise NotImplementedError( - "cannot convert ref (e.g. HEAD) -> commit id" - ) from err - # core function ---- - iterations = await _get_project_iterations_range( - vc_repo, _project_uuid, commit_id, offset=_offset, limit=_limit + iterations_range = await _get_project_iterations_range( + vc_repo, + meta_project_uuid, + meta_project_commit_id, + offset=q.offset, + limit=q.limit, ) - if iterations.total_count == 0: + if iterations_range.total_count == 0: raise web.HTTPNotFound( - reason=f"No iterations found for project {_project_uuid=}/{commit_id=}" + reason=f"No iterations found for project {meta_project_uuid=}/{meta_project_commit_id=}" ) - assert len(iterations.items) <= _limit # nosec + assert len(iterations_range.items) <= q.limit # nosec # parse and validate response ---- page_items = [ - ProjectIterationAsItem( - name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", - parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), - workcopy_project_id=wcp_id, - workcopy_project_url=url_for( - "get_project", - project_id=wcp_id, - ), - url=url_for( - f"{__name__}._list_meta_project_iterations_handler", - project_uuid=_project_uuid, - ref_id=commit_id, - ), + ProjectIterationItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + url_for, ) - for wcp_id, iter_id in iterations.items + for item in iterations_range.items ] - page = Page[ProjectIterationAsItem].parse_obj( + page = Page[ProjectIterationItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, - total=iterations.total_count, - limit=_limit, - offset=_offset, + total=iterations_range.total_count, + limit=q.limit, + offset=q.offset, ) ) return web.Response( @@ -250,51 +346,129 @@ async def _create_meta_project_iterations_handler(request: web.Request) -> web.R # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) - _project_uuid = ProjectID(request.match_info["project_uuid"]) - _ref_id = request.match_info["ref_id"] - try: - commit_id = CommitID(_ref_id) - except ValueError as err: - # e.g. HEAD - raise NotImplementedError( - "cannot convert ref (e.g. HEAD) -> commit id" - ) from err - # core function ---- project_iterations = await create_or_get_project_iterations( - vc_repo, _project_uuid, commit_id + vc_repo, meta_project_uuid, meta_project_commit_id ) # parse and validate response ---- iterations_items = [ - ProjectIterationAsItem( - name=f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", - parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), - workcopy_project_id=wcp_id, - workcopy_project_url=url_for( - "get_project", - project_id=wcp_id, - ), - url=url_for( - f"{__name__}._create_meta_project_iterations_handler", - project_uuid=_project_uuid, - ref_id=commit_id, - ), + ProjectIterationItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + url_for, ) - for wcp_id, iter_id in project_iterations + for item in project_iterations ] return envelope_json_response(iterations_items, web.HTTPCreated) # TODO: registry as route when implemented. Currently iteration is retrieved via GET /projects/{workcopy_project_id} -# @routes.get( -# f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}", -# name=f"{__name__}._get_meta_project_iterations_handler", -# ) -# SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}", + name=f"{__name__}._get_meta_project_iterations_handler", +) +@login_required +@permission_required("project.snapshot.read") async def _get_meta_project_iterations_handler(request: web.Request) -> web.Response: - raise NotImplementedError + raise NotImplementedError( + "SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735" + ) + + +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/-/results", + name=f"{__name__}._list_meta_project_iterations_results_handler", +) +@login_required +@permission_required("project.snapshot.read") +async def _list_meta_project_iterations_results_handler( + request: web.Request, +) -> web.Response: + # parse and validate request ---- + q = parse_query_parameters(request) + meta_project_uuid = q.project_uuid + meta_project_commit_id = q.ref_id + + url_for = create_url_for_function(request) + vc_repo = VersionControlForMetaModeling(request) + + # core function ---- + iterations_range = await _get_project_iterations_range( + vc_repo, + meta_project_uuid, + meta_project_commit_id, + offset=q.offset, + limit=q.limit, + ) + + if iterations_range.total_count == 0: + raise web.HTTPNotFound( + reason=f"No iterations found for projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}" + ) + + assert len(iterations_range.items) <= q.limit # nosec + + # get every project from the database and extract results + _prj_data = {} + for item in iterations_range.items: + # TODO: fetch ALL project iterations at once. Otherwise they will have different results + # TODO: if raises? + prj = await vc_repo.get_project(f"{item.project_id}", include=["workbench"]) + _prj_data[item.project_id] = prj["workbench"] + + def _get_project_results(project_id) -> ExtractedResults: + # TODO: if raises? + results = extract_project_results(_prj_data[project_id]) + return results + + # parse and validate response ---- + page_items = [ + ProjectIterationResultItem.create( + meta_project_uuid, + meta_project_commit_id, + item.iteration_index, + item.project_id, + _get_project_results(item.project_id), + url_for, + ) + for item in iterations_range.items + ] + + page = Page[ProjectIterationResultItem].parse_obj( + paginate_data( + chunk=page_items, + request_url=request.url, + total=iterations_range.total_count, + limit=q.limit, + offset=q.offset, + ) + ) + return web.Response( + text=page.json(**RESPONSE_MODEL_POLICY), + content_type="application/json", + ) + + +@routes.get( + f"/{VTAG}/projects/{{project_uuid}}/checkpoint/{{ref_id}}/iterations/{{iter_id}}/results", + name=f"{__name__}._get_meta_project_iteration_results_handler", +) +@login_required +@permission_required("project.snapshot.read") +async def _get_meta_project_iteration_results_handler( + request: web.Request, +) -> web.Response: + raise NotImplementedError( + "SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735" + ) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py index 28f05e4b3f89..82c1f20967df 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_iterations.py @@ -17,6 +17,7 @@ from models_library.services import ServiceDockerData from pydantic import BaseModel, ValidationError from pydantic.fields import Field +from pydantic.types import PositiveInt from .meta_modeling_function_nodes import ( FUNCTION_SERVICE_TO_CALLABLE, @@ -124,6 +125,8 @@ def extract_parameters( # DOMAIN MODEL for project iteration ------------------------------------------------------------ +IterationID = PositiveInt + class ProjectIteration(BaseModel): """ @@ -135,10 +138,16 @@ class ProjectIteration(BaseModel): # version-control info repo_id: Optional[int] = None - repo_commit_id: CommitID = Field(...) + repo_commit_id: CommitID = Field( + ..., + description="this id makes it unique but does not guarantees order. See iter_index for that", + ) # iteration info - iter_index: int = Field(...) + iteration_index: IterationID = Field( + ..., + description="Index that allows iterations to be sortable", + ) total_count: Union[int, Literal["unbound"]] = "unbound" parameters_checksum: SHA1Str = Field(...) @@ -159,7 +168,7 @@ def to_tag_name(self) -> str: """Composes unique tag name for this iteration""" return compose_iteration_tag_name( repo_commit_id=self.repo_commit_id, - iter_index=self.iter_index, + iteration_index=self.iteration_index, total_count=self.total_count, parameters_checksum=self.parameters_checksum, ) @@ -168,22 +177,20 @@ def to_tag_name(self) -> str: # NOTE: compose_/parse_ functions are basically serialization functions for ProjectIteration # into/from string tags. An alternative approach would be simply using json.dump/load # but we should guarantee backwards compatibilty with old tags -# +# TODO: change this by json-serialization def compose_iteration_tag_name( repo_commit_id: CommitID, - iter_index: int, + iteration_index: IterationID, total_count: Union[int, str], parameters_checksum: SHA1Str, ) -> str: """Composes unique tag name for iter_index-th iteration of repo_commit_id out of total_count""" - return ( - f"iteration:{repo_commit_id}/{iter_index}/{total_count}/{parameters_checksum}" - ) + return f"iteration:{repo_commit_id}/{iteration_index}/{total_count}/{parameters_checksum}" def parse_iteration_tag_name(name: str) -> Dict[str, Any]: if m := re.match( - r"^iteration:(?P\d+)/(?P\d+)/(?P-*\d+)/(?P.*)$", + r"^iteration:(?P\d+)/(?P\d+)/(?P-*\d+)/(?P.*)$", name, ): return m.groupdict() @@ -261,14 +268,16 @@ async def get_or_create_runnable_projects( total_count = len(iterations) original_name = project["name"] - for iter_index, (parameters, updated_nodes) in enumerate(iterations): + # FIXME: in an optimization, iteration_index should start with LAST iterated index + for iteration_index, (parameters, updated_nodes) in enumerate(iterations, start=1): log.debug( - "Creating snapshot of project %s with parameters=%s", - project_uuid, - parameters, + "Creating snapshot of project %s with parameters=%s [%s]", + f"{project_uuid=}", + f"{parameters=}", + f"{updated_nodes=}", ) - project["name"] = f"{original_name}/{iter_index}" + project["name"] = f"{original_name}/{iteration_index}" project["workbench"].update( { # converts model in dict patching first thumbnail @@ -282,7 +291,7 @@ async def get_or_create_runnable_projects( project_iteration = ProjectIteration( repo_id=repo_id, repo_commit_id=main_commit_id, - iter_index=iter_index, + iteration_index=iteration_index, total_count=total_count, parameters_checksum=_compute_params_checksum(parameters), ) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py new file mode 100644 index 000000000000..4191686fa8bb --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_results.py @@ -0,0 +1,111 @@ +""" Access to the to projects module + + - Adds a middleware to intercept /projects/* requests + - Implements a MetaProjectRunPolicy policy (see director_v2_abc.py) to define how meta-projects run + +""" + + +import logging +from typing import Any, Dict + +from models_library.projects_nodes import Outputs +from models_library.projects_nodes_io import NodeIDStr +from pydantic import BaseModel, Field, conint + +log = logging.getLogger(__name__) + + +class ExtractedResults(BaseModel): + progress: Dict[NodeIDStr, conint(ge=0, le=100)] = Field( + ..., description="Progress in each computational node" + ) + labels: Dict[NodeIDStr, str] = Field( + ..., description="Maps captured node with a label" + ) + values: Dict[NodeIDStr, Outputs] = Field( + ..., description="Captured outputs per node" + ) + + class Config: + schema_extra = { + "example": { + # sample with 2 computational services, 2 data sources (iterator+parameter) and 2 observers (probes) + "progress": { + "4c08265a-427b-4ac3-9eab-1d11c822ada4": 0, + "e33c6880-1b1d-4419-82d7-270197738aa9": 100, + }, + "labels": { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": "Integer iterator", + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": "Probe Sensor - Integer", + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": "Probe Sensor - Integer_2", + "d76fca06-f050-4790-88a8-0aac10c87b39": "Boolean Parameter", + }, + "values": { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": { + "out_1": 1, + "out_2": [3, 4], + }, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": {"in_1": 7}, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, + "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, + }, + } + } + + +def extract_project_results(workbench: Dict[str, Any]) -> ExtractedResults: + """Extracting results from a project's workbench section (i.e. pipeline). Specifically: + + - data sources (e.g. outputs from iterators, paramters) + - progress of evaluators (e.g. a computational service) + - data observers (basically inputs from probes) + + NOTE: all projects produces from iterations preserve the same node uuids so + running this extraction on all projects from a iterations allows to create a + row for a table of results + """ + # nodeid -> % progress + progress = {} + # nodeid -> label (this map is necessary because cannot guaratee labels to be unique) + labels = {} + # nodeid -> { port: value , ...} # results have two levels deep: node/port + results = {} + + for noid, node in workbench.items(): + key_parts = node["key"].split("/") + + # evaluate progress + if "comp" in key_parts: + progress[noid] = node.get("progress", 0) + + # evaluate results + if "probe" in key_parts: + label = node["label"] + values = {} + for port_name, node_input in node["inputs"].items(): + try: + values[port_name] = workbench[node_input["nodeUuid"]]["outputs"][ + node_input["output"] + ] + except KeyError: + # if not run, we know name but NOT value + values[port_name] = "n/a" + results[noid], labels[noid] = values, label + + elif "data-iterator" in key_parts: + label = node["label"] + try: + values = node["outputs"] # {oid: value, ...} + except KeyError: + # if not iterated, we do not know NEITHER name NOT values + values = {} + results[noid], labels[noid] = values, label + + elif "parameter" in key_parts: + label = node["label"] + values = node["outputs"] + results[noid], labels[noid] = values, label + + res = ExtractedResults(progress=progress, labels=labels, values=results) + return res diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py b/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py index aff42f0efd1e..cad0b62a425f 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling_version_control.py @@ -43,32 +43,35 @@ async def get_workcopy_project(self, repo_id: int, commit_id: int) -> ProjectDic assert project # nosec return dict(project.items()) - async def get_project(self, project_id: ProjectIDStr) -> ProjectDict: + async def get_project( + self, project_id: ProjectIDStr, *, include: Optional[List[str]] = None + ) -> ProjectDict: async with self.engine.acquire() as conn: if self.user_id is None: raise UserUndefined() + if include is None: + include = [ + "type", + "uuid", + "name", + "description", + "thumbnail", + "prj_owner", + "access_rights", + "workbench", + "ui", + "classifiers", + "dev", + "quality", + "published", + "hidden", + ] + project = ( await self.ProjectsOrm(conn) - .set_filter(uuid=str(project_id), prj_owner=self.user_id) - .fetch( - [ - "type", - "uuid", - "name", - "description", - "thumbnail", - "prj_owner", - "access_rights", - "workbench", - "ui", - "classifiers", - "dev", - "quality", - "published", - "hidden", - ] - ) + .set_filter(uuid=f"{project_id}", prj_owner=self.user_id) + .fetch(include) ) assert project # nosec project_as_dict = dict(project.items()) diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py index eba7bc2374bd..e48f9fe4ac51 100644 --- a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_iterations.py @@ -22,7 +22,8 @@ from simcore_service_webserver.director_v2_api import get_project_run_policy from simcore_service_webserver.meta_modeling_handlers import ( Page, - ProjectIterationAsItem, + ProjectIterationItem, + ProjectIterationResultItem, ) from simcore_service_webserver.meta_modeling_projects import ( meta_project_policy, @@ -138,7 +139,7 @@ async def _mock_start(project_id, user_id, **options): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() - first_iterlist = Page[ProjectIterationAsItem].parse_obj(body).data + first_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(first_iterlist) == 3 @@ -172,6 +173,16 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): # ---------------------------------------------- + # GET results of all iterations + # /projects/{project_uuid}/checkpoint/{ref_id}/iterations/-/results + resp = await client.get( + f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations/-/results" + ) + assert resp.status == HTTPStatus.OK, await resp.text() + body = await resp.json() + + results = Page[ProjectIterationResultItem].parse_obj(body).data + # GET project and MODIFY iterator values---------------------------------------------- # - Change iterations from 0:4 -> HEAD+1 resp = await client.get(f"/v0/projects/{project_uuid}") @@ -228,7 +239,8 @@ async def _mock_catalog_get(app, user_id, product_name, only_key_versions): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await resp.json() - second_iterlist = Page[ProjectIterationAsItem].parse_obj(body).data + assert resp.status == HTTPStatus.OK, f"{body=}" # nosec + second_iterlist = Page[ProjectIterationItem].parse_obj(body).data assert len(second_iterlist) == 4 assert len(set(it.workcopy_project_id for it in second_iterlist)) == len( diff --git a/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py new file mode 100644 index 000000000000..f1e78358e24d --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/10/meta_modeling/test_meta_modeling_results.py @@ -0,0 +1,138 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import json +from typing import Any, Dict, Type + +import pytest +from pydantic import BaseModel +from simcore_service_webserver.meta_modeling_results import ( + ExtractedResults, + extract_project_results, +) + + +@pytest.fixture +def fake_workbench() -> Dict[str, Any]: + return { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": { + "key": "simcore/services/frontend/data-iterator/funky-range", + "version": "1.0.0", + "label": "Integer iterator", + "inputs": {"linspace_start": 0, "linspace_stop": 2, "linspace_step": 1}, + "inputNodes": [], + # some funky output of iterator/param, + "outputs": {"out_1": 1, "out_2": [3, 4]}, + }, + "e33c6880-1b1d-4419-82d7-270197738aa9": { + "key": "simcore/services/comp/itis/sleeper", + "version": "2.0.0", + "label": "sleeper", + "inputs": { + "input_2": { + "nodeUuid": "0f1e38c9-dcb7-443c-a745-91b97ac28ccc", + "output": "out_1", + }, + "input_3": False, + }, + "inputNodes": ["0f1e38c9-dcb7-443c-a745-91b97ac28ccc"], + "state": { + "currentStatus": "SUCCESS", + "modified": False, + "dependencies": [], + }, + "progress": 100, + "outputs": { + "output_1": { + "store": "0", + "path": "30359da5-ca4d-3288-a553-5f426a204fe6/e33c6880-1b1d-4419-82d7-270197738aa9/single_number.txt", + "eTag": "a87ff679a2f3e71d9181a67b7542122c", + }, + "output_2": 7, + }, + "runHash": "f92d1836aa1b6b1b031f9e1b982e631814708675c74ba5f02161e0f256382b2b", + }, + "4c08265a-427b-4ac3-9eab-1d11c822ada4": { + "key": "simcore/services/comp/itis/sleeper", + "version": "2.0.0", + "label": "sleeper", + "inputNodes": [], + }, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": { + "key": "simcore/services/frontend/iterator-consumer/probe/int", + "version": "1.0.0", + "label": "Probe Sensor - Integer", + "inputs": { + "in_1": { + "nodeUuid": "e33c6880-1b1d-4419-82d7-270197738aa9", + "output": "output_2", + } + }, + "inputNodes": ["e33c6880-1b1d-4419-82d7-270197738aa9"], + }, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": { + "key": "simcore/services/frontend/iterator-consumer/probe/int", + "version": "1.0.0", + "label": "Probe Sensor - Integer_2", + "inputs": { + "in_1": { + "nodeUuid": "0f1e38c9-dcb7-443c-a745-91b97ac28ccc", + "output": "out_1", + } + }, + "inputNodes": ["0f1e38c9-dcb7-443c-a745-91b97ac28ccc"], + }, + "d76fca06-f050-4790-88a8-0aac10c87b39": { + "key": "simcore/services/frontend/parameter/boolean", + "version": "1.0.0", + "label": "Boolean Parameter", + "inputs": {}, + "inputNodes": [], + "outputs": {"out_1": True}, + }, + } + + +def test_extract_project_results(fake_workbench: Dict[str, Any]): + + results = extract_project_results(fake_workbench) + + print(json.dumps(results.progress, indent=1)) + print(json.dumps(results.labels, indent=1)) + print(json.dumps(results.values, indent=1)) + + # this has to be something that shall be deployable in a table + assert results.progress == { + "4c08265a-427b-4ac3-9eab-1d11c822ada4": 0, + "e33c6880-1b1d-4419-82d7-270197738aa9": 100, + } + + # labels are not unique, so there is a map to nodeids + assert results.labels == { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": "Integer iterator", + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": "Probe Sensor - Integer", + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": "Probe Sensor - Integer_2", + "d76fca06-f050-4790-88a8-0aac10c87b39": "Boolean Parameter", + } + # this is basically a tree that defines columns + assert results.values == { + "0f1e38c9-dcb7-443c-a745-91b97ac28ccc": {"out_1": 1, "out_2": [3, 4]}, + "2d0ce8b9-c9c3-43ce-ad2f-ad493898de37": {"in_1": 7}, + "445b44d1-59b3-425c-ac48-7c13e0f2ea5b": {"in_1": 1}, + "d76fca06-f050-4790-88a8-0aac10c87b39": {"out_1": True}, + } + + +@pytest.mark.parametrize( + "model_cls", + (ExtractedResults,), +) +def test_models_examples( + model_cls: Type[BaseModel], model_cls_examples: Dict[str, Any] +): + for name, example in model_cls_examples.items(): + print(name, ":", json.dumps(example, indent=1)) + model_instance = model_cls(**example) + assert model_instance, f"Failed with {name}"