From 09c200d5ec164a8f9f90df888cad26259ca3fc50 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 17 Mar 2022 18:28:50 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=91=EF=B8=8F=20hotfix=20GC=20first=20d?= =?UTF-8?q?eploy=20(#2899)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...age-collector => .env-wb-garbage-collector | 0 .github/CODEOWNERS | 3 ++- services/docker-compose.devel.yml | 2 +- services/docker-compose.local.yml | 2 +- services/docker-compose.yml | 4 +-- .../garbage_collector.py | 7 +++-- .../projects/plugin.py | 14 ++-------- .../projects/project_models.py | 26 ++++++++++++++++++- .../projects/projects_db.py | 7 +++-- 9 files changed, 43 insertions(+), 22 deletions(-) rename .env-webserver-garbage-collector => .env-wb-garbage-collector (100%) diff --git a/.env-webserver-garbage-collector b/.env-wb-garbage-collector similarity index 100% rename from .env-webserver-garbage-collector rename to .env-wb-garbage-collector diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cf314678103..49af9fe22f7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,7 @@ # files and folders recursively .codeclimate.yml @sanderegg @pcrespov -.env-* @pcrespov +.env-* @pcrespov @Surfict @mrnicegyu11 .travis.yml @sanderegg Makefile @pcrespov @sanderegg @@ -26,6 +26,7 @@ Makefile @pcrespov @sanderegg /services/api-server/ @pcrespov /services/catalog/ @pcrespov @sanderegg /services/director*/ @sanderegg @pcrespov +/services/docker-compose*.yml @sanderegg @Surfict @mrnicegyu11 /services/dynamic-sidecar/ @GitHK /services/migration/ @pcrespov /services/static-webserver @GitHK diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index 29169a0a9d4..7c0838729cb 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -69,7 +69,7 @@ services: SC_BOOT_MODE: debug-ptvsd WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-DEBUG} - webserver-garbage-collector: + wb-garbage-collector: volumes: *webserver-volumes-dev environment: <<: *webserver-environment-dev diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index d3b800f9ca7..b6a3fcaeec4 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -71,7 +71,7 @@ services: - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.priority=3 - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry - webserver-garbage-collector: + wb-garbage-collector: environment: SC_BOOT_MODE: ${SC_BOOT_MODE:-default} WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-INFO} diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 99a31a3aa23..b720560e5bf 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -238,7 +238,7 @@ services: - default - interactive_services_subnet - webserver-garbage-collector: + wb-garbage-collector: image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} init: true hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" @@ -247,7 +247,7 @@ services: WEBSERVER_GARBAGE_COLLECTOR: '{"GARBAGE_COLLECTOR_INTERVAL_S": 30}' env_file: - ../.env - - ../.env-webserver-garbage-collector + - ../.env-wb-garbage-collector healthcheck: test: diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector.py b/services/web/server/src/simcore_service_webserver/garbage_collector.py index 2dc872a1f6e..b550ffa48bb 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector.py @@ -4,7 +4,7 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from .garbage_collector_task import run_background_task -from .projects.projects_db import setup_projects_db +from .projects.plugin import setup_projects_db, setup_projects_model_schema logger = logging.getLogger(__name__) @@ -17,7 +17,10 @@ ) def setup_garbage_collector(app: web.Application): - ## project-api needs access to db + ## needs a partial init of projects plugin since this plugin uses projects-api + # - project-api needs access to db setup_projects_db(app) + # - projects-api needs access to schema + setup_projects_model_schema(app) app.cleanup_ctx.append(run_background_task) diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 6fdad1b3582..3f8acdba2a8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -3,12 +3,10 @@ A project is a document defining a osparc study It contains metadata about the study (e.g. name, description, owner, etc) and a workbench section that describes the study pipeline """ -import json import logging from pprint import pformat from aiohttp import web -from servicelib.aiohttp.application_keys import APP_JSONSCHEMA_SPECS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from servicelib.aiohttp.rest_routing import ( get_handlers_from_namespace, @@ -17,8 +15,8 @@ ) from .._constants import APP_OPENAPI_SPECS_KEY, APP_SETTINGS_KEY -from .._resources import resources from . import projects_handlers, projects_nodes_handlers, projects_tags_handlers +from .project_models import setup_projects_model_schema from .projects_access import setup_projects_access from .projects_db import setup_projects_db @@ -87,13 +85,5 @@ def setup_projects(app: web.Application) -> bool: # app.router.add_routes( _create_routes("node", specs, nodes_handlers) ) # json-schemas for projects datasets - # FIXME: schemas are hard-coded to api/V0!!! - with resources.stream("api/v0/schemas/project-v0.0.1.json") as fh: - project_schema = json.load(fh) - - if APP_JSONSCHEMA_SPECS_KEY in app: - app[APP_JSONSCHEMA_SPECS_KEY]["projects"] = project_schema - else: - app[APP_JSONSCHEMA_SPECS_KEY] = {"projects": project_schema} - + setup_projects_model_schema(app) return True diff --git a/services/web/server/src/simcore_service_webserver/projects/project_models.py b/services/web/server/src/simcore_service_webserver/projects/project_models.py index 8d479345bdd..ed1703ab80a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/project_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/project_models.py @@ -1,9 +1,14 @@ +import json from enum import Enum from typing import Any, Dict, Optional, Tuple +from aiohttp import web from aiopg.sa.result import RowProxy from simcore_postgres_database.models.projects import ProjectType +from .._constants import APP_JSONSCHEMA_SPECS_KEY +from .._resources import resources + # TODO: extend ProjectDict = Dict[str, Any] ProjectProxy = RowProxy @@ -23,4 +28,23 @@ def to_project_type_db(cls, api_type: "ProjectTypeAPI") -> Optional[ProjectType] }[api_type] -__all__: Tuple[str, ...] = ("ProjectDict", "ProjectProxy") +def setup_projects_model_schema(app: web.Application): + # NOTE: inits once per app + # FIXME: schemas are hard-coded to api/V0!!! + with resources.stream("api/v0/schemas/project-v0.0.1.json") as fh: + project_schema = json.load(fh) + + if app.get(APP_JSONSCHEMA_SPECS_KEY) is None: + app[APP_JSONSCHEMA_SPECS_KEY] = {"projects": project_schema} + + elif app[APP_JSONSCHEMA_SPECS_KEY].get("projects") is None: + app[APP_JSONSCHEMA_SPECS_KEY]["projects"] = project_schema + + return app[APP_JSONSCHEMA_SPECS_KEY]["projects"] + + +__all__: Tuple[str, ...] = ( + "ProjectDict", + "ProjectProxy", + "setup_projects_model_schema", +) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_db.py b/services/web/server/src/simcore_service_webserver/projects/projects_db.py index a535337335c..6c0a5053431 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_db.py @@ -866,5 +866,8 @@ async def update_project_without_checking_permissions( def setup_projects_db(app: web.Application): - db = ProjectDBAPI(app) - app[APP_PROJECT_DBAPI] = db + # NOTE: inits once per app + if app.get(APP_PROJECT_DBAPI) is None: + db = ProjectDBAPI(app) + app[APP_PROJECT_DBAPI] = db + return app[APP_PROJECT_DBAPI]