From f1a97c1680e4c9b7a6b46aee7166f90d983bcc1e Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Thu, 12 May 2022 14:20:05 -0400 Subject: [PATCH 01/14] prepare magpie/twitcher/weaver requests hooks for WPS outputs in user workspace directory (relates to https://github.com/Ouranosinc/Magpie/pull/517 and https://github.com/bird-house/twitcher/pull/114) --- CHANGES.md | 18 +++++++++++++-- .../weaver/config/magpie/adapter_hooks.py | 22 +++++++++++++++++++ .../weaver/config/magpie/config.yml.template | 20 +++++++++++++++++ .../weaver/docker-compose-extra.yml | 9 ++++++++ birdhouse/default.env | 2 +- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 birdhouse/components/weaver/config/magpie/adapter_hooks.py diff --git a/CHANGES.md b/CHANGES.md index a2e419cfe..f609e2d16 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,22 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes: + +- Magpie/Twitcher: update `magpie` service + from [3.21.0](https://github.com/Ouranosinc/Magpie/tree/3.21.0) + to [3.25.0](https://github.com/Ouranosinc/Magpie/tree/3.25.0) and + bundled `twitcher` from [0.6.2](https://github.com/bird-house/twitcher/tree/v0.6.2) + to [0.7.0](https://github.com/bird-house/twitcher/tree/v0.7.0). + + - Adds [Service Hooks](https://pavics-magpie.readthedocs.io/en/latest/configuration.html#service-hooks) allowing + Twitcher to apply HTTP pre-request/post-response modifications to requested services and resources in accordance + to `MagpieAdapter` implementation and using plugin Python scripts when matched against specific request parameters. + + - Using *Service Hooks*, inject ``X-WPS-Output-Context`` header in Weaver job submission requests through the proxied + request by Twitcher and `MagpieAdapter`. This header contains the user ID that indicates to Weaver were to store + job output results, allowing to save them in the corresponding user's workspace directory under `wpsoutputs` path. + [1.18.12](https://github.com/bird-house/birdhouse-deploy/tree/1.18.12) (2022-05-05) ------------------------------------------------------------------------------------------------------------------ @@ -3314,4 +3329,3 @@ Prior Versions All versions prior to [1.7.0](https://github.com/bird-house/birdhouse-deploy/tree/1.7.0) were not officially tagged. Is it strongly recommended employing later versions to ensure better traceability of changes that could impact behavior and potential issues on new server instances. - diff --git a/birdhouse/components/weaver/config/magpie/adapter_hooks.py b/birdhouse/components/weaver/config/magpie/adapter_hooks.py new file mode 100644 index 000000000..484903ed9 --- /dev/null +++ b/birdhouse/components/weaver/config/magpie/adapter_hooks.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pyramid.request import Request + + from magpie.typedefs import ServiceConfigItem + + +def add_x_wps_output_context(request, service): + # type: (Request, ServiceConfigItem) -> Request + if "application/json" in request.content_type: + body = request.json # JSON generated from body, cannot override directly + # following for testing purposes only + body["hooks"] = len(service["hooks"]) + body["hook"] = "add_x_wps_output_context" + request.body = json.dumps(body).encode() + if request.user is not None: + request.headers["X-WPS-Output-Context"] = "user-" + str(request.user.id) + return request diff --git a/birdhouse/components/weaver/config/magpie/config.yml.template b/birdhouse/components/weaver/config/magpie/config.yml.template index 85d821038..73be2f19f 100644 --- a/birdhouse/components/weaver/config/magpie/config.yml.template +++ b/birdhouse/components/weaver/config/magpie/config.yml.template @@ -10,6 +10,26 @@ providers: c4i: false type: api # FIXME: 'ades' when https://github.com/Ouranosinc/Magpie/issues/360 implemented sync_type: api + # see following for hooks details: + # - https://github.com/Ouranosinc/Magpie/blob/master/config/providers.cfg + # - https://pavics-magpie.readthedocs.io/en/latest/configuration.html#service-hooks + hooks: + # when a job is created in weaver, apply the header that will nest output results under user's context directory + # see also: + # - https://pavics-weaver.readthedocs.io/en/latest/processes.html?highlight=x-wps-output-context#outputs-location + # each path below are equivalents, but with more or less specific reference to the requested service/process + - type: request + path: "/providers/[\\w_-]+/processes/[\\w_-]+/jobs" + method: POST + target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context + - type: request + path: "/processes/[\\w_-]+/jobs" + method: POST + target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context + - type: request + path: "/jobs" + method: POST + target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env' ${WEAVER_WPS_NAME}: diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml index 2e16bb60f..aae2863ab 100644 --- a/birdhouse/components/weaver/docker-compose-extra.yml +++ b/birdhouse/components/weaver/docker-compose-extra.yml @@ -30,6 +30,15 @@ services: - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/permissions/weaver-permissions.cfg:ro - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/providers/weaver-provider.cfg:ro + # extend twitcher with MagpieAdapter hooks employed for weaver proxied requests + twitcher: + environment: + MAGPIE_CONFIG_PATH: /opt/birdhouse/src/magpie/config.yml + volumes: + # NOTE: MagpieAdapter hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them + - ./components/weaver/config/magpie/config.yml:/opt/birdhouse/src/magpie/config.yml + - ./components/weaver/config/magpie/adapter_hooks.py:/opt/birdhouse/src/magpie/adapter_hooks.py + # Image 'weaver' is the API side of the application weaver: container_name: ${WEAVER_MANAGER_NAME} diff --git a/birdhouse/default.env b/birdhouse/default.env index c35ff8995..426b47732 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -22,7 +22,7 @@ export GEOSERVER_IMAGE="pavics/geoserver:2.19.0-kartoza-build20210329" export BASH_IMAGE="bash:5.1.4" # Tag version that will be used to update Magpie API, Magpie CLI, and matching Twitcher with Magpie Adapter -export MAGPIE_VERSION=3.21.0 +export MAGPIE_VERSION=3.25.0 # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" From e1c3f736a4bb2da38e6fe940fdfe4f455a39f958 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Fri, 20 May 2022 17:14:42 -0400 Subject: [PATCH 02/14] fix invalid paths + remove test code + add more details/case handling --- .../weaver/config/magpie/adapter_hooks.py | 47 ++++++++++++++----- .../weaver/config/magpie/config.yml.template | 7 +-- .../weaver/docker-compose-extra.yml | 2 +- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/birdhouse/components/weaver/config/magpie/adapter_hooks.py b/birdhouse/components/weaver/config/magpie/adapter_hooks.py index 484903ed9..b9c8afa86 100644 --- a/birdhouse/components/weaver/config/magpie/adapter_hooks.py +++ b/birdhouse/components/weaver/config/magpie/adapter_hooks.py @@ -1,22 +1,45 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json +""" +These hooks will be running within Twitcher, using MagpieAdapter context. + +The code below can make use of any package that is installed by Magpie/Twitcher. +""" + from typing import TYPE_CHECKING +from magpie.constants import get_constant +from magpie.utils import get_header + if TYPE_CHECKING: from pyramid.request import Request - from magpie.typedefs import ServiceConfigItem + +def is_admin(request): + # type: (Request) -> bool + admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) + return admin_group in [group.group_name for group in request.user.groups] -def add_x_wps_output_context(request, service): - # type: (Request, ServiceConfigItem) -> Request - if "application/json" in request.content_type: - body = request.json # JSON generated from body, cannot override directly - # following for testing purposes only - body["hooks"] = len(service["hooks"]) - body["hook"] = "add_x_wps_output_context" - request.body = json.dumps(body).encode() - if request.user is not None: - request.headers["X-WPS-Output-Context"] = "user-" + str(request.user.id) +def add_x_wps_output_context(request): + # type: (Request) -> Request + """ + Apply the ``X-WPS-Output-Context`` for saving outputs in the user-context WPS-outputs directory. + """ + header = get_header("X-WPS-Output-Context", request.headers) + # if explicitly provided, ensure it is permitted (admin allow any, otherwise self-user reference only) + if header is not None: + if request.user is None: + header = "public" + else: + if not is_admin(request): + # override disallowed writing to other location + # otherwise, up to admin to have writen something sensible + header = "user-" + str(request.user.id) + else: + if request.user is None: + header = "public" + else: + header = "user-" + str(request.user.id) + request.headers["X-WPS-Output-Context"] = header return request diff --git a/birdhouse/components/weaver/config/magpie/config.yml.template b/birdhouse/components/weaver/config/magpie/config.yml.template index 73be2f19f..29ffadbb9 100644 --- a/birdhouse/components/weaver/config/magpie/config.yml.template +++ b/birdhouse/components/weaver/config/magpie/config.yml.template @@ -10,6 +10,7 @@ providers: c4i: false type: api # FIXME: 'ades' when https://github.com/Ouranosinc/Magpie/issues/360 implemented sync_type: api + # hook locations should be relative to mounted Twitcher location as they are run within that container # see following for hooks details: # - https://github.com/Ouranosinc/Magpie/blob/master/config/providers.cfg # - https://pavics-magpie.readthedocs.io/en/latest/configuration.html#service-hooks @@ -21,15 +22,15 @@ providers: - type: request path: "/providers/[\\w_-]+/processes/[\\w_-]+/jobs" method: POST - target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context + target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context - type: request path: "/processes/[\\w_-]+/jobs" method: POST - target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context + target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context - type: request path: "/jobs" method: POST - target: /opt/local/src/magpie/hooks/adapter_hooks.py:add_x_wps_output_context + target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env' ${WEAVER_WPS_NAME}: diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml index aae2863ab..04ab7e1ba 100644 --- a/birdhouse/components/weaver/docker-compose-extra.yml +++ b/birdhouse/components/weaver/docker-compose-extra.yml @@ -37,7 +37,7 @@ services: volumes: # NOTE: MagpieAdapter hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them - ./components/weaver/config/magpie/config.yml:/opt/birdhouse/src/magpie/config.yml - - ./components/weaver/config/magpie/adapter_hooks.py:/opt/birdhouse/src/magpie/adapter_hooks.py + - ./components/weaver/config/magpie/weaver_hooks.py:/opt/birdhouse/src/magpie/hooks/weaver_hooks.py # Image 'weaver' is the API side of the application weaver: From 85e3faa39b043889879405bb1209536ec766c42a Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Fri, 20 May 2022 17:44:02 -0400 Subject: [PATCH 03/14] fix wrong file name for weaver hooks --- .../weaver/config/magpie/{adapter_hooks.py => weaver_hooks.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename birdhouse/components/weaver/config/magpie/{adapter_hooks.py => weaver_hooks.py} (97%) diff --git a/birdhouse/components/weaver/config/magpie/adapter_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py similarity index 97% rename from birdhouse/components/weaver/config/magpie/adapter_hooks.py rename to birdhouse/components/weaver/config/magpie/weaver_hooks.py index b9c8afa86..a1458adf1 100644 --- a/birdhouse/components/weaver/config/magpie/adapter_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -These hooks will be running within Twitcher, using MagpieAdapter context. +These hooks will be running within Twitcher, using MagpieAdapter context, applied for Weaver requests. The code below can make use of any package that is installed by Magpie/Twitcher. """ From 9470caf9f4e9e6ea9238a50154829a456579e2b0 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Fri, 20 May 2022 21:57:09 -0400 Subject: [PATCH 04/14] weaver process filtering implementation --- .gitignore | 4 ++ .../weaver/config/magpie/config.yml.template | 4 ++ .../weaver/config/magpie/weaver_hooks.py | 63 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/.gitignore b/.gitignore index 655efcb35..7307b17a2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ ubuntu-bionic-18.04-cloudimg-console.log ## vim temp files *.swp + +## Python temp files +**/__pycache__ +**/*.py[cod] diff --git a/birdhouse/components/weaver/config/magpie/config.yml.template b/birdhouse/components/weaver/config/magpie/config.yml.template index 29ffadbb9..c8594a6b9 100644 --- a/birdhouse/components/weaver/config/magpie/config.yml.template +++ b/birdhouse/components/weaver/config/magpie/config.yml.template @@ -31,6 +31,10 @@ providers: path: "/jobs" method: POST target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:add_x_wps_output_context + - type: response + path: "/processes" + method: GET + target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:filter_allowed_processes # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env' ${WEAVER_WPS_NAME}: diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index a1458adf1..59d73c761 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -6,13 +6,19 @@ The code below can make use of any package that is installed by Magpie/Twitcher. """ +import json from typing import TYPE_CHECKING +from magpie.api.management.resource import resource_utils as ru from magpie.constants import get_constant +from magpie.permissions import Access, Permission from magpie.utils import get_header if TYPE_CHECKING: from pyramid.request import Request + from pyramid.response import Response + + from magpie.adapter import HookContext def is_admin(request): @@ -43,3 +49,60 @@ def add_x_wps_output_context(request): header = "user-" + str(request.user.id) request.headers["X-WPS-Output-Context"] = header return request + + +def filter_allowed_processes(response, context): + # type: (Response, HookContext) -> Response + """ + Filter processes returned by Weaver response according to allowed resources by user. + """ + if "application/json" in response.content_type: + body = response.json + if "processes" in body: + if is_admin(response.request): # don't waste time checking permissions, full access anyway + return response + + # depending on 'detail' query, processes can be returned as list of IDs or nested JSON summaries + processes = { + proc if isinstance(proc, str) else proc.get("id"): proc + for proc in body["processes"] + } + + # only need 2 first levels ('processes' and each process 'id' under it) + children = ru.get_resource_children(context.resource, response.request.db, limit_depth=2) + proc_res = None + for res in children.values(): + if res["node"] == "processes": + # if nothing under 'processes' resource, then guarantee no permissions, done check + if not res["children"]: + return response + proc_res = res + break + if not proc_res: + return response # 'processes' itself does not exist, no permissions possible and done check + + allowed_processes = [] + known_processes = proc_res["children"].values() + known_processes = {res["node"].resource.resource_name: res for res in known_processes} + for proc_name in processes: + if proc_name not in known_processes: + continue # do not bother checking missing resource + child_proc = known_processes[proc_name] + perms = context.service.effective_permissions(response.request.user, child_proc, [Permission.READ]) + if perms[0].access == Access.ALLOW: + proc = processes[proc_name] + allowed_processes.append(proc) + + # override collected and permitted processes access by user + body["processes"] = allowed_processes + + # WARNING: + # JSON generated from 'body' attribute cannot be overridden directly (computed inline). + # Also, since we override, must set any Content header accordingly with modifications. + data = json.dumps(body).encode("UTF-8") + response.body = data + c_len = len(data) + response.content_length = c_len + response.headers["Content-Length"] = str(c_len) + + return response From 180ee5e4b8d3ba58ecac7047bcd9e634f4f4bd98 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Tue, 24 May 2022 11:33:01 -0400 Subject: [PATCH 05/14] adjust permission and node parsing for weaver processes filtered by allowed permission to read them --- .../components/weaver/config/magpie/config.yml.template | 3 ++- birdhouse/components/weaver/config/magpie/weaver_hooks.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/birdhouse/components/weaver/config/magpie/config.yml.template b/birdhouse/components/weaver/config/magpie/config.yml.template index c8594a6b9..f55f02d6e 100644 --- a/birdhouse/components/weaver/config/magpie/config.yml.template +++ b/birdhouse/components/weaver/config/magpie/config.yml.template @@ -91,9 +91,10 @@ permissions: action: create # Process deployment (write) and listing (read) + # use 'read-match' to allow only listing, and not describe underlying processes (require 'read' on them individually) - service: ${WEAVER_MANAGER_NAME} resource: /processes # GET is processes listing, POST is deploy: only allow view by anonymous - permission: read # under '/processes/...', JSON 'DescribeProcess', POST job submit, GET results, etc. + permission: read-match # under '/processes/...', JSON 'DescribeProcess', POST job submit, GET results, etc. group: anonymous action: create diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index 59d73c761..59c109e55 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -72,7 +72,7 @@ def filter_allowed_processes(response, context): children = ru.get_resource_children(context.resource, response.request.db, limit_depth=2) proc_res = None for res in children.values(): - if res["node"] == "processes": + if res["node"].resource_name == "processes": # if nothing under 'processes' resource, then guarantee no permissions, done check if not res["children"]: return response @@ -83,11 +83,11 @@ def filter_allowed_processes(response, context): allowed_processes = [] known_processes = proc_res["children"].values() - known_processes = {res["node"].resource.resource_name: res for res in known_processes} + known_processes = {res["node"].resource_name: res for res in known_processes} for proc_name in processes: if proc_name not in known_processes: continue # do not bother checking missing resource - child_proc = known_processes[proc_name] + child_proc = known_processes[proc_name]["node"] perms = context.service.effective_permissions(response.request.user, child_proc, [Permission.READ]) if perms[0].access == Access.ALLOW: proc = processes[proc_name] From 2af0d5db89587eed5c135f29cf82e5ff02b57b4b Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Tue, 24 May 2022 12:25:03 -0400 Subject: [PATCH 06/14] fix filter process in case of anonymous (no user authenticated) --- birdhouse/components/weaver/config/magpie/weaver_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index 59c109e55..f239a7ae6 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from magpie.api.management.resource import resource_utils as ru +from magpie.api.requests import get_user from magpie.constants import get_constant from magpie.permissions import Access, Permission from magpie.utils import get_header @@ -24,6 +25,8 @@ def is_admin(request): # type: (Request) -> bool admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) + if not request.user: # no user authenticated (public) + return False return admin_group in [group.group_name for group in request.user.groups] @@ -84,11 +87,12 @@ def filter_allowed_processes(response, context): allowed_processes = [] known_processes = proc_res["children"].values() known_processes = {res["node"].resource_name: res for res in known_processes} + request_user = get_user(response.request) for proc_name in processes: if proc_name not in known_processes: continue # do not bother checking missing resource child_proc = known_processes[proc_name]["node"] - perms = context.service.effective_permissions(response.request.user, child_proc, [Permission.READ]) + perms = context.service.effective_permissions(request_user, child_proc, [Permission.READ]) if perms[0].access == Access.ALLOW: proc = processes[proc_name] allowed_processes.append(proc) From 9b9e4e8e58b8691362d67fa9cb5fd0b256e67885 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Tue, 24 May 2022 12:33:22 -0400 Subject: [PATCH 07/14] bump magpie 3.26.0 to provide context paramter in response hook --- CHANGES.md | 5 ++++- birdhouse/default.env | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f609e2d16..dff0de2c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ - Magpie/Twitcher: update `magpie` service from [3.21.0](https://github.com/Ouranosinc/Magpie/tree/3.21.0) - to [3.25.0](https://github.com/Ouranosinc/Magpie/tree/3.25.0) and + to [3.26.0](https://github.com/Ouranosinc/Magpie/tree/3.26.0) and bundled `twitcher` from [0.6.2](https://github.com/bird-house/twitcher/tree/v0.6.2) to [0.7.0](https://github.com/bird-house/twitcher/tree/v0.7.0). @@ -30,6 +30,9 @@ request by Twitcher and `MagpieAdapter`. This header contains the user ID that indicates to Weaver were to store job output results, allowing to save them in the corresponding user's workspace directory under `wpsoutputs` path. + - Using *Service Hooks*, filter processes returned by Weaver in JSON response from ``/processes`` endpoint using + respective permissions applied onto each ``/processes/{processID}`` for the requesting user. Users will only be able + to see processes for which they have read access to retrieve the process description. [1.18.12](https://github.com/bird-house/birdhouse-deploy/tree/1.18.12) (2022-05-05) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/default.env b/birdhouse/default.env index 426b47732..214887eb0 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -22,7 +22,7 @@ export GEOSERVER_IMAGE="pavics/geoserver:2.19.0-kartoza-build20210329" export BASH_IMAGE="bash:5.1.4" # Tag version that will be used to update Magpie API, Magpie CLI, and matching Twitcher with Magpie Adapter -export MAGPIE_VERSION=3.25.0 +export MAGPIE_VERSION=3.26.0 # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" From 40d6ff9535c76f0a49a6f1f53cecb21a0f35ad02 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Thu, 26 May 2022 12:41:10 -0400 Subject: [PATCH 08/14] adjust weaver request/response hooks volume-mounts to load MagpieAdapter configs (within Twitcher) using directory loading strategy --- .../components/weaver/docker-compose-extra.yml | 18 ++++++++---------- birdhouse/docker-compose.yml | 11 ++++++++++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml index 04ab7e1ba..84d16c375 100644 --- a/birdhouse/components/weaver/docker-compose-extra.yml +++ b/birdhouse/components/weaver/docker-compose-extra.yml @@ -23,21 +23,19 @@ services: magpie: volumes: # NOTE: - # Although file use the "config.yml" format, it is very important to pass it as independent/duplicate reference - # provider/permissions ".cfg" files. This is because Magpie will not parse multiple "config.yml" files - # additively with other component's ".cfg" files, as "config.yml" are intended for unique-combined definitions. - # Data structure within "config.yml" is the same as within the respective sections in typical ".cfg" files. - - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/permissions/weaver-permissions.cfg:ro - - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/providers/weaver-provider.cfg:ro + # Although file uses the "config.yml" format, it is very important to pass it as independent/duplicate reference + # provider/permissions config files. This is because 'MAGPIE_CONFIG_PATH' is not used to allow parsing multiple + # config files for each extendable service, using loading of all configuration files found in mount directories. + - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/permissions/weaver-permissions.yml:ro + - ./components/weaver/config/magpie/config.yml:/opt/local/src/magpie/config/providers/weaver-provider.yml:ro # extend twitcher with MagpieAdapter hooks employed for weaver proxied requests twitcher: - environment: - MAGPIE_CONFIG_PATH: /opt/birdhouse/src/magpie/config.yml volumes: # NOTE: MagpieAdapter hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them - - ./components/weaver/config/magpie/config.yml:/opt/birdhouse/src/magpie/config.yml - - ./components/weaver/config/magpie/weaver_hooks.py:/opt/birdhouse/src/magpie/hooks/weaver_hooks.py + # target mount location depends on main docker-compose 'MAGPIE_PROVIDERS_CONFIG_PATH' environment variable + - ./components/weaver/config/magpie/config.yml:/opt/birdhouse/src/magpie/config/weaver-config.yml:ro + - ./components/weaver/config/magpie/weaver_hooks.py:/opt/birdhouse/src/magpie/hooks/weaver_hooks.py:ro # Image 'weaver' is the API side of the application weaver: diff --git a/birdhouse/docker-compose.yml b/birdhouse/docker-compose.yml index 121e2678b..7dc212f14 100644 --- a/birdhouse/docker-compose.yml +++ b/birdhouse/docker-compose.yml @@ -333,7 +333,8 @@ services: environment: TWITCHER_PROTECTED_URL: https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH} # target directories to allow loading multiple config files of corresponding category - # each compose override should volume mount its files in the matching directories + # each compose override should volume mount its files inside the matching directories + # (note: DO NOT use 'MAGPIE_CONFIG_PATH' that would disable multi-config loading capability) MAGPIE_PROVIDERS_CONFIG_PATH: "/opt/local/src/magpie/config/providers" MAGPIE_PERMISSIONS_CONFIG_PATH: "/opt/local/src/magpie/config/permissions" MAGPIE_POSTGRES_HOST: postgres-magpie @@ -358,6 +359,14 @@ services: container_name: twitcher ports: - "8000:8000" + environment: + # target directories to allow loading multiple config files of corresponding category + # each compose override should volume mount its files inside the below directory + # (note: DO NOT use 'MAGPIE_CONFIG_PATH' that would disable multi-config loading capability) + # Only 'providers' sections are used to employ 'request/response hooks' with 'MagpieAdapter'. + # Hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them. + # Other Magpie components are unknown and ignored by Twitcher itself. + MAGPIE_PROVIDERS_CONFIG_PATH: "/opt/birdhouse/src/magpie/config" env_file: - ./config/postgres-magpie/credentials.env depends_on: From 28ad95c5ffb0aa1208c7996797fb114ed56d341c Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Thu, 26 May 2022 17:41:15 -0400 Subject: [PATCH 09/14] document sample JSON body for weaver processes listing used in hooks --- .../weaver/config/magpie/weaver_hooks.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index f239a7ae6..fe40d8119 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -58,6 +58,53 @@ def filter_allowed_processes(response, context): # type: (Response, HookContext) -> Response """ Filter processes returned by Weaver response according to allowed resources by user. + + Following are sample (clipped) JSON body that can be expected from Weaver (or any OGC API - Processes). + + Using ``GET https:///processes`` + + .. code-block:: json + :caption: Detailed process listing from Weaver (other fields than 'processes' are removed for concise example). + + { + "processes": [ + { + "id": "ColibriFlyingpigeon_SubsetBbox", + "title": "ColibriFlyingpigeon_SubsetBbox", + "mutable": true, + "keywords": [ + "application" + ], + "metadata": [], + "jobControlOptions": [ + "async-execute" + ], + "outputTransmission": [ + "reference", + "value" + ], + "processDescriptionURL": "https:///processes/ColibriFlyingpigeon_SubsetBbox", + "processEndpointWPS1": "https:///ows/wps", + "executeEndpoint": "https:///processes/ColibriFlyingpigeon_SubsetBbox/jobs" + } + ] + } + + Using ``GET https:///processes?detail=false`` + + .. code-block:: json + :caption: Simple process listing from Weaver (other fields than 'processes' are removed for concise example). + + { + "description": "Listing of available processes successful.", + "processes": [ + "CatFile", + "ColibriFlyingpigeon_SubsetBbox", + ], + "page": 0, + "total": 2 + } + """ if "application/json" in response.content_type: body = response.json From 5c4f9b28e8cea144842add7524048931fe5b52c0 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Thu, 26 May 2022 17:45:26 -0400 Subject: [PATCH 10/14] documentation reference link about hooks definition and usage --- birdhouse/components/weaver/config/magpie/weaver_hooks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index fe40d8119..59e2e8ff0 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -4,6 +4,10 @@ These hooks will be running within Twitcher, using MagpieAdapter context, applied for Weaver requests. The code below can make use of any package that is installed by Magpie/Twitcher. + +.. seealso:: + Documentation about Magpie/Twitcher request/response hooks is available here: + https://pavics-magpie.readthedocs.io/en/latest/configuration.html#service-hooks """ import json From e661ef269bdeba5d8a503bfbc5727b8c8082964e Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Mon, 6 Jun 2022 17:20:00 -0400 Subject: [PATCH 11/14] auto-create user perms for deployed weaver process --- .../weaver/config/magpie/config.yml.template | 4 + .../weaver/config/magpie/weaver_hooks.py | 124 +++++++++++++++++- 2 files changed, 125 insertions(+), 3 deletions(-) diff --git a/birdhouse/components/weaver/config/magpie/config.yml.template b/birdhouse/components/weaver/config/magpie/config.yml.template index f55f02d6e..6298aca70 100644 --- a/birdhouse/components/weaver/config/magpie/config.yml.template +++ b/birdhouse/components/weaver/config/magpie/config.yml.template @@ -35,6 +35,10 @@ providers: path: "/processes" method: GET target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:filter_allowed_processes + - type: response + path: "/processes" + method: POST + target: /opt/birdhouse/src/magpie/hooks/weaver_hooks.py:allow_user_deployed_processes # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env' ${WEAVER_WPS_NAME}: diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index 59e2e8ff0..860355320 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -13,11 +13,15 @@ import json from typing import TYPE_CHECKING +import transaction + from magpie.api.management.resource import resource_utils as ru -from magpie.api.requests import get_user +from magpie.api.management.user import user_utils as uu +from magpie.api.requests import get_user, get_service_matchdict_checked from magpie.constants import get_constant -from magpie.permissions import Access, Permission -from magpie.utils import get_header +from magpie.models import Route +from magpie.permissions import Access, Permission, PermissionSet, Scope +from magpie.utils import get_header, get_logger if TYPE_CHECKING: from pyramid.request import Request @@ -25,6 +29,8 @@ from magpie.adapter import HookContext +LOGGER = get_logger("birdhouse-weaver-hooks") + def is_admin(request): # type: (Request) -> bool @@ -161,3 +167,115 @@ def filter_allowed_processes(response, context): response.headers["Content-Length"] = str(c_len) return response + + +def allow_user_deployed_processes(response): + # type: (Response) -> Response + """ + Add the user permissions to read (listing and description) and execute the process for the deploying user. + + This will grant access to the process definition by the user that deployed it until they desire to make it public. + At a later time, a request to the appropriate group to share (restricted group) or to publish publicly (anonymous) + could be made to create the relevant permissions to describe or execute the process by other users. + + Expected response format from service: + + .. code-block:: json + { + "processSummary": { + "id": "", + "..." + } + } + + If any failure occurs, simply return the response to let deployment succeed, but user will not receive access to it + automatically. Manual update of permissions would be necessary by platform administrator via Magpie. + """ + p_id = "" + u_name = "" + try: + # only apply permission if deployment was successful + if "application/json" in response.content_type and response.status_code == 201: + body = response.json + info = body.get("processSummary", {}) or body.get("process", {}) # bw-compat + p_id = info.get("id") + if not (p_id and isinstance(p_id, str)): + return response + + # user is not necessarily admin + # in fact, this operation is only needed if non-admin, since admin has full access anyway + request = response.request + if is_admin(request): + return response + user = request.user + # if deploy endpoint was made public, then even anonymous could deploy (not recommended, but possible) + if not user: + user = get_user(request) + u_name = user.user_name + + # note: matchdict reference of Twitcher owsproxy view is used, just so happens to be same name as Magpie + service = get_service_matchdict_checked(request) + + # find the nested resource matching: "weaver/processes/" + children = ru.get_resource_children(service, request.db, limit_depth=2) + p_res = None + for res in children.values(): + if res["node"].resource_name == "processes": + processes_res_id = res["node"].resource_id + for child_res in res["children"].values(): + if child_res["node"].resource_name == p_id: + p_res = child_res["node"] + break + break + else: + # resource 'processes' should already exist, but create it if somehow missing + # otherwise, it will be impossible to create '' under it + resp = ru.create_resource("processes", None, Route.resource_type_name, service.resource_id, request.db) + processes_res_id = resp.json["resource"]["resource_id"] + + # note: + # since this is running within a *response* hook, the request transaction is already handled + # define a new transaction to create new resources + with transaction.manager: + + # if '' somehow already exists, use it + if p_res is None: + resp = ru.create_resource(p_id, None, Route.resource_type_name, processes_res_id, request.db) + p_res_id = resp.json["resource"]["resource_id"] + p_res = ru.ResourceService.by_resource_id(p_res_id, request.db) + if not p_res: + LOGGER.warning( + "Failed creation of permissions for user [%s] to access deployed process [%s] in Weaver. " + "Could not retrieve resource matching deployed process!", u_name, p_id + ) + return response + + # apply necessary permissions to give full access to the deployed process to the user + # override permissions to undo what could have been previously applied (only if already existed) + p_desc = PermissionSet(Permission.READ, Access.ALLOW, Scope.RECURSIVE) # describe proc + jobs statuses + p_exec = PermissionSet(Permission.WRITE, Access.ALLOW, Scope.RECURSIVE) # edit process + execute jobs + r_desc = uu.create_user_resource_permission_response(user, p_res, p_desc, request.db, overwrite=True) + r_exec = uu.create_user_resource_permission_response(user, p_res, p_exec, request.db, overwrite=True) + + # summit transaction results (new resources and permissions) + transaction.commit() + + # sanity check + if r_desc.status_code in [200, 201] and r_exec.status_code in [200, 201]: + LOGGER.info( + "Successful creation of permissions for user [%s] to access deployed process [%s] in Weaver.", + u_name, p_id + ) + else: + statuses = [r_desc.status_code, r_exec.status_code] + LOGGER.warning( + "Failed creation of permissions for user [%s] to access deployed process [%s] in Weaver. " + "Permission creation returned unexpected statuses: %s", u_name, p_id, statuses + ) + except Exception as exc: + LOGGER.error( + "Failed creation of permissions for user [%s] to access deployed process [%s] in Weaver. " + "Unexpected exception occurred: [%s]", u_name, p_id, str(exc) + ) + + return response From 1ccc3c178930d5af4b5057f02db1e2fafc82c6c4 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Tue, 7 Jun 2022 13:58:33 -0400 Subject: [PATCH 12/14] update changes --- CHANGES.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index dff0de2c2..fc4638abd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,10 +29,19 @@ - Using *Service Hooks*, inject ``X-WPS-Output-Context`` header in Weaver job submission requests through the proxied request by Twitcher and `MagpieAdapter`. This header contains the user ID that indicates to Weaver were to store job output results, allowing to save them in the corresponding user's workspace directory under `wpsoutputs` path. + More details found in PR https://github.com/bird-house/birdhouse-deploy/pull/244. - Using *Service Hooks*, filter processes returned by Weaver in JSON response from ``/processes`` endpoint using respective permissions applied onto each ``/processes/{processID}`` for the requesting user. Users will only be able to see processes for which they have read access to retrieve the process description. + More details found in PR https://github.com/bird-house/birdhouse-deploy/pull/245. + + - Using *Service Hooks*, automatically apply permissions for the user that successfully deployed a Weaver process + using ``POST /processes`` request, granting it direct access to this process during process listing, process + description request and for submitting job execution of this process. + Only this user deploying the process will have access to it until further permissions are added in Magpie to share + or publish it with other users, groups and/or publicly. The user must have the necessary permission to deploy a new + process in the first place. More details found in PR https://github.com/bird-house/birdhouse-deploy/pull/247. [1.18.12](https://github.com/bird-house/birdhouse-deploy/tree/1.18.12) (2022-05-05) ------------------------------------------------------------------------------------------------------------------ From 52e7d63fbeba4a343d4b82a09ba8df7ae955a0d5 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Tue, 7 Jun 2022 15:12:35 -0400 Subject: [PATCH 13/14] fix typo --- birdhouse/components/weaver/config/magpie/weaver_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/components/weaver/config/magpie/weaver_hooks.py b/birdhouse/components/weaver/config/magpie/weaver_hooks.py index 860355320..045cace3b 100644 --- a/birdhouse/components/weaver/config/magpie/weaver_hooks.py +++ b/birdhouse/components/weaver/config/magpie/weaver_hooks.py @@ -53,7 +53,7 @@ def add_x_wps_output_context(request): else: if not is_admin(request): # override disallowed writing to other location - # otherwise, up to admin to have writen something sensible + # otherwise, up to admin to have written something sensible header = "user-" + str(request.user.id) else: if request.user is None: From c67c00339f122b7c72995c9d1e0086e17a550b51 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Wed, 8 Jun 2022 18:36:59 -0400 Subject: [PATCH 14/14] =?UTF-8?q?Bump=20version:=201.18.13=20=E2=86=92=201?= =?UTF-8?q?.19.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGES.md | 5 +++++ README.rst | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4ba20a407..7869e0a77 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.18.13 +current_version = 1.19.0 commit = True tag = False tag_name = {new_version} diff --git a/CHANGES.md b/CHANGES.md index 0d93860d8..3fe9d7d2c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.19.0](https://github.com/bird-house/birdhouse-deploy/tree/1.19.0) (2022-06-08) +------------------------------------------------------------------------------------------------------------------ + ## Changes: - Magpie/Twitcher: update `magpie` service diff --git a/README.rst b/README.rst index 48ca857e2..4b1495d7c 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.18.13.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.19.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.18.13...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.19.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.18.13-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.19.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.18.13 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.19.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version)