From 8373c0fe6d5e24ea1ccaa8f6792ae9331a08097e Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 10:51:59 +0200 Subject: [PATCH 01/17] Use create_lock instead of lock_publication in post --- src/layman/layer/rest_workspace_layers.py | 2 +- src/layman/map/rest_workspace_maps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layman/layer/rest_workspace_layers.py b/src/layman/layer/rest_workspace_layers.py index 26a8fb714..4f9c1cc6d 100644 --- a/src/layman/layer/rest_workspace_layers.py +++ b/src/layman/layer/rest_workspace_layers.py @@ -126,7 +126,7 @@ def post(workspace): filenames = [f.filename for f in files] input_file.check_filenames(workspace, layername, filenames, check_crs) - redis_util.lock_publication(workspace, LAYER_TYPE, layername, request.method) + redis_util.create_lock(workspace, LAYER_TYPE, layername, 19, request.method) try: # register layer uuid diff --git a/src/layman/map/rest_workspace_maps.py b/src/layman/map/rest_workspace_maps.py index 974ead727..7b0365971 100644 --- a/src/layman/map/rest_workspace_maps.py +++ b/src/layman/map/rest_workspace_maps.py @@ -75,7 +75,7 @@ def post(workspace): mapurl = url_for('rest_workspace_map.get', mapname=mapname, workspace=workspace) - redis_util.lock_publication(workspace, MAP_TYPE, mapname, request.method) + redis_util.create_lock(workspace, MAP_TYPE, mapname, 29, request.method) try: map_result = { From 1a0ca14d60ced33dfb6d4674c2d7e5dea791f848 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 10:53:25 +0200 Subject: [PATCH 02/17] Rename TASK_ID_TO_PUBLICATION to LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION --- src/layman/celery.py | 10 +++++----- src/layman/uuid.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/layman/celery.py b/src/layman/celery.py index 9ef1d5142..dc1b4f95d 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -7,7 +7,7 @@ REDIS_CURRENT_TASK_NAMES = f"{__name__}:CURRENT_TASK_NAMES" PUBLICATION_CHAIN_INFOS = f'{__name__}:PUBLICATION_TASK_INFOS' -TASK_ID_TO_PUBLICATION = f'{__name__}:TASK_ID_TO_PUBLICATION' +LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION = f'{__name__}:TASK_ID_TO_PUBLICATION' def task_prerun(workspace, _publication_type, publication_name, _task_id, task_name): @@ -25,7 +25,7 @@ def task_postrun(workspace, publication_type, publication_name, task_id, task_na task_hash = _get_task_hash(task_name, workspace, publication_name) rds.srem(key, task_hash) - key = TASK_ID_TO_PUBLICATION + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION hash = task_id if rds.hexists(key, hash): finnish_publication_task(task_id) @@ -42,7 +42,7 @@ def _get_task_hash(task_name, workspace, publication_name): def finnish_publication_task(task_id): rds = settings.LAYMAN_REDIS - key = TASK_ID_TO_PUBLICATION + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION hash = task_id publ_hash = rds.hget(key, hash) if publ_hash is None: @@ -132,7 +132,7 @@ def set_publication_chain_info(workspace, publication_type, publication_name, ta set_publication_chain_info_dict(workspace, publication_type, publication_name, chain_info) rds = settings.LAYMAN_REDIS - key = TASK_ID_TO_PUBLICATION + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION val = _get_publication_hash(workspace, publication_type, publication_name) hash = chain_info['last'] rds.hset(key, hash, val) @@ -198,7 +198,7 @@ def delete_publication(workspace, publication_type, publication_name): hash = _get_publication_hash(workspace, publication_type, publication_name) rds.hdel(key, hash) - key = TASK_ID_TO_PUBLICATION + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION rds.hdel(key, task_id) diff --git a/src/layman/uuid.py b/src/layman/uuid.py index 9600f7e57..ed8c5a406 100644 --- a/src/layman/uuid.py +++ b/src/layman/uuid.py @@ -163,7 +163,7 @@ def check_redis_consistency(expected_publ_num_by_type=None): t for t in task_names_tuples if t[1] == username and t[2] == pubname and t[0].startswith(publ_type_name) ), None) is None) is is_ready, f"{username}, {publ_type_name}, {pubname}: {is_ready}, {task_names_tuples}" - assert (redis.hget(celery_util.TASK_ID_TO_PUBLICATION, chain_info['last'].task_id) is None) is is_ready + assert (redis.hget(celery_util.LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION, chain_info['last'].task_id) is None) is is_ready # publication locks locks = redis.hgetall(redis_util.PUBLICATION_LOCKS_KEY) From 81fe06ec4e3db6018aeaa4ffd0e88d54db38a45e Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 11:16:03 +0200 Subject: [PATCH 03/17] For WFS-T request lock layers just before patch_after_wfst --- src/layman/geoserver_proxy.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/layman/geoserver_proxy.py b/src/layman/geoserver_proxy.py index a977ef85b..35fcd88d1 100644 --- a/src/layman/geoserver_proxy.py +++ b/src/layman/geoserver_proxy.py @@ -179,13 +179,10 @@ def proxy(subpath): headers_req[settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE] = authn_username app.logger.info(f"{request.method} GeoServer proxy, headers_req={headers_req}, url={url}") - changing_wfs_t_layers = set() + wfs_t_layers = set() if data is not None and len(data) > 0: try: wfs_t_attribs, wfs_t_layers = extract_attributes_and_layers_from_wfs_t(data) - changing_wfs_t_layers = {(workspace, layer) for workspace, layer in wfs_t_layers if authz.can_i_edit(LAYER_TYPE, workspace, layer)} - for workspace, layer in changing_wfs_t_layers: - redis.create_lock(workspace, LAYER_TYPE, layer, 19, 'wfst') if wfs_t_attribs: ensure_wfs_t_attributes(wfs_t_attribs) except BaseException as err: @@ -198,8 +195,10 @@ def proxy(subpath): allow_redirects=False ) - for workspace, layername in changing_wfs_t_layers: - patch_after_wfst(workspace, layername) + for workspace, layername in wfs_t_layers: + if authz.can_i_edit(LAYER_TYPE, workspace, layername): + redis.create_lock(workspace, LAYER_TYPE, layername, 19, 'wfst') + patch_after_wfst(workspace, layername) excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] headers = {key: value for (key, value) in response.headers.items() if key.lower() not in excluded_headers} From 6806563aefa1d04a81450a10841cfb4040fe9575 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 11:42:46 +0200 Subject: [PATCH 04/17] Refactor, use request methods as constants instead of hard coded on each place --- src/layman/authz/__init__.py | 15 ++++++++------- src/layman/celery.py | 4 ++-- src/layman/common/__init__.py | 10 ++++++++++ src/layman/common/redis.py | 24 ++++++++++++++---------- src/layman/geoserver_proxy.py | 4 ++-- src/layman/layer/micka/csw.py | 8 ++++---- src/layman/layer/micka/tasks.py | 10 +++++----- src/layman/layer/rest_workspace_layer.py | 5 +++-- src/layman/layer/util.py | 4 ++-- src/layman/map/micka/csw.py | 10 +++++----- src/layman/map/micka/tasks.py | 10 +++++----- 11 files changed, 60 insertions(+), 44 deletions(-) diff --git a/src/layman/authz/__init__.py b/src/layman/authz/__init__.py index 456fc4a9a..15c9ad606 100644 --- a/src/layman/authz/__init__.py +++ b/src/layman/authz/__init__.py @@ -2,7 +2,7 @@ from functools import wraps from flask import after_this_request, request -from layman import LaymanError, settings, authn, util as layman_util +from layman import LaymanError, settings, authn, util as layman_util, common from layman.common.prime_db_schema import workspaces, users from layman.common.rest import parse_request_path @@ -16,11 +16,11 @@ def authorize(workspace, publication_type, publication_name, request_method, act }[publication_type] if is_multi_publication_request: - if request_method in ['GET', 'DELETE']: + if request_method.lower() in [common.REQUEST_METHOD_GET, common.REQUEST_METHOD_DELETE]: if not workspaces.get_workspace_infos(workspace): raise LaymanError(40) # Workspace not found return - if request_method in ['POST']: + if request_method.lower() in [common.REQUEST_METHOD_POST]: if actor_name == workspace: return if ((not users.get_user_infos(workspace)) # public workspace @@ -45,11 +45,12 @@ def authorize(workspace, publication_type, publication_name, request_method, act if not publ_info: raise LaymanError(publication_not_found_code) user_can_read = is_user_in_access_rule(actor_name, publ_info['access_rights']['read']) - if request_method in ['GET']: + if request_method.lower() in [common.REQUEST_METHOD_GET]: if user_can_read: return raise LaymanError(publication_not_found_code) - if request_method in ['POST', 'PUT', 'PATCH', 'DELETE']: + if request_method.lower() in [common.REQUEST_METHOD_PATCH, common.REQUEST_METHOD_DELETE, + common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PUT, ]: if is_user_in_access_rule(actor_name, publ_info['access_rights']['write']): return if user_can_read: @@ -113,7 +114,7 @@ def decorated_function(*args, **kwargs): actor_name = authn.get_authn_username() # raises exception in case of unauthorized request authorize(workspace, publication_type, publication_name, request.method, actor_name) - if workspace and publication_type and not publication_name and request.method == 'GET': + if workspace and publication_type and not publication_name and request.method == common.REQUEST_METHOD_GET: # pylint: disable=unused-variable @after_this_request def authorize_after_request_tmp(response): @@ -132,7 +133,7 @@ def decorated_function(*args, **kwargs): if publication_type is None or workspace or publication_name: raise Exception(f"Authorization module is unable to authorize path {req_path}") actor_name = authn.get_authn_username() - if request.method == 'GET': + if request.method == common.REQUEST_METHOD_GET: # pylint: disable=unused-variable @after_this_request def authorize_after_request_tmp(response): diff --git a/src/layman/celery.py b/src/layman/celery.py index dc1b4f95d..585670a4f 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -2,7 +2,7 @@ from flask import current_app from celery.contrib.abortable import AbortableAsyncResult -from layman import settings +from layman import settings, common from layman.common import redis as redis_util REDIS_CURRENT_TASK_NAMES = f"{__name__}:CURRENT_TASK_NAMES" @@ -56,7 +56,7 @@ def finnish_publication_task(task_id): rds.hdel(key, hash) lock = redis_util.get_publication_lock(username, publication_type, publication_name) - if lock in ['patch', 'post', 'wfst', ]: + if lock in [common.REQUEST_METHOD_PATCH, common.REQUEST_METHOD_POST, common.PUBLICATION_LOCK_CODE_WFST, ]: redis_util.unlock_publication(username, publication_type, publication_name) diff --git a/src/layman/common/__init__.py b/src/layman/common/__init__.py index 41b506e3b..a11be0dc3 100644 --- a/src/layman/common/__init__.py +++ b/src/layman/common/__init__.py @@ -1,5 +1,15 @@ from collections import namedtuple +REQUEST_METHOD_DELETE = 'delete' +REQUEST_METHOD_GET = 'get' +REQUEST_METHOD_PATCH = 'patch' +REQUEST_METHOD_POST = 'post' +REQUEST_METHOD_PUT = 'put' +PUBLICATION_LOCK_CODE_POST = 'post' +PUBLICATION_LOCK_CODE_PATCH = 'patch' +PUBLICATION_LOCK_CODE_DELETE = 'delete' +PUBLICATION_LOCK_CODE_WFST = 'wfst' + InternalSourceTypeDef = namedtuple('InternalSourceTypeDef', ['info_items', ]) diff --git a/src/layman/common/redis.py b/src/layman/common/redis.py index 98573a13f..fbcacb752 100644 --- a/src/layman/common/redis.py +++ b/src/layman/common/redis.py @@ -1,7 +1,7 @@ from functools import wraps from flask import request, current_app -from layman import settings, celery as celery_util +from layman import settings, celery as celery_util, common from layman import LaymanError PUBLICATION_LOCKS_KEY = f'{__name__}:PUBLICATION_LOCKS' @@ -70,22 +70,26 @@ def solve_locks(workspace, publication_type, publication_name, error_code, metho ) if current_lock is None: return - if method not in ['patch', 'delete', 'wfst', ]: + if method not in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_DELETE, + common.PUBLICATION_LOCK_CODE_WFST, ]: raise Exception(f"Unknown method to check: {method}") - if current_lock not in ['patch', 'delete', 'post', 'wfst', ]: + if current_lock not in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_DELETE, + common.PUBLICATION_LOCK_CODE_POST, + common.PUBLICATION_LOCK_CODE_WFST, ]: raise Exception(f"Unknown current lock: {current_lock}") - if current_lock in ['patch', 'post']: - if method in ['patch', 'post']: + if current_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: + if method in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: raise LaymanError(error_code) - elif current_lock in ['delete']: - if method in ['patch', 'post']: + elif current_lock in [common.PUBLICATION_LOCK_CODE_DELETE, ]: + if method in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: raise LaymanError(error_code) - if method not in ['delete']: - if (current_lock, method) == ('wfst', 'wfst'): + if method not in [common.PUBLICATION_LOCK_CODE_DELETE, ]: + if (current_lock, method) == (common.PUBLICATION_LOCK_CODE_WFST, common.PUBLICATION_LOCK_CODE_WFST): chain_info = celery_util.get_publication_chain_info(workspace, publication_type, publication_name) celery_util.abort_chain(chain_info) else: - assert current_lock not in ['wfst', ] and method not in ['wfst', ],\ + assert current_lock not in [common.PUBLICATION_LOCK_CODE_WFST, ] and method not in [ + common.PUBLICATION_LOCK_CODE_WFST, ],\ f'current_lock={current_lock}, method={method},' \ f'workspace, publication_type, publication_name={(workspace, publication_type, publication_name)}' diff --git a/src/layman/geoserver_proxy.py b/src/layman/geoserver_proxy.py index 35fcd88d1..d95c74b07 100644 --- a/src/layman/geoserver_proxy.py +++ b/src/layman/geoserver_proxy.py @@ -7,7 +7,7 @@ from flask import Blueprint, g, current_app as app, request, Response from geoserver.util import reset as gs_reset -from layman import authn, authz, settings +from layman import authn, authz, common, settings from layman.authn import authenticate, is_user_with_name from layman.common import redis from layman.layer import db, LAYER_TYPE, util as layer_util @@ -197,7 +197,7 @@ def proxy(subpath): for workspace, layername in wfs_t_layers: if authz.can_i_edit(LAYER_TYPE, workspace, layername): - redis.create_lock(workspace, LAYER_TYPE, layername, 19, 'wfst') + redis.create_lock(workspace, LAYER_TYPE, layername, 19, common.PUBLICATION_LOCK_CODE_WFST) patch_after_wfst(workspace, layername) excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] diff --git a/src/layman/layer/micka/csw.py b/src/layman/layer/micka/csw.py index f1292ded9..b887afb21 100644 --- a/src/layman/layer/micka/csw.py +++ b/src/layman/layer/micka/csw.py @@ -14,7 +14,7 @@ from layman.layer.geoserver import wms from layman.layer.geoserver import wfs from layman.layer import LAYER_TYPE -from layman import settings, patch_mode, LaymanError +from layman import settings, patch_mode, LaymanError, common from layman.util import url_for, get_publication_info PATCH_MODE = patch_mode.NO_DELETE @@ -65,7 +65,7 @@ def patch_layer(workspace, layername, metadata_properties_to_refresh, _actor_nam return None # current_app.logger.info(f"Current element=\n{ET.tostring(el, encoding='unicode', pretty_print=True)}") - _, prop_values = get_template_path_and_values(workspace, layername, http_method='patch') + _, prop_values = get_template_path_and_values(workspace, layername, http_method=common.REQUEST_METHOD_PATCH) prop_values = { k: v for k, v in prop_values.items() if k in metadata_properties_to_refresh + ['md_date_stamp'] @@ -113,7 +113,7 @@ def csw_insert(workspace, layername): def get_template_path_and_values(workspace, layername, http_method=None): - assert http_method in ['post', 'patch'] + assert http_method in [common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH] publ_info = get_publication_info(workspace, LAYER_TYPE, layername, context={ 'keys': ['title', 'bounding_box', 'description'], }) @@ -161,7 +161,7 @@ def get_template_path_and_values(workspace, layername, http_method=None): scale_denominator=scale_denominator, epsg_codes=settings.LAYMAN_OUTPUT_SRS_LIST, ) - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: prop_values.pop('revision_date', None) template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'record-template.xml') return template_path, prop_values diff --git a/src/layman/layer/micka/tasks.py b/src/layman/layer/micka/tasks.py index 1b6e3b24d..f6ebd7bec 100644 --- a/src/layman/layer/micka/tasks.py +++ b/src/layman/layer/micka/tasks.py @@ -1,7 +1,7 @@ from celery.utils.log import get_task_logger from layman.celery import AbortedException -from layman import celery_app +from layman import celery_app, common from layman.common import empty_method_returns_true from . import csw, soap @@ -16,11 +16,11 @@ bind=True, base=celery_app.AbortableTask ) -def refresh_csw(self, username, layername, http_method='post', metadata_properties_to_refresh=None): +def refresh_csw(self, username, layername, http_method=common.REQUEST_METHOD_POST, metadata_properties_to_refresh=None): metadata_properties_to_refresh = metadata_properties_to_refresh or [] if self.is_aborted(): raise AbortedException - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: csw.csw_insert(username, layername) else: csw.patch_layer(username, layername, metadata_properties_to_refresh) @@ -35,11 +35,11 @@ def refresh_csw(self, username, layername, http_method='post', metadata_properti bind=True, base=celery_app.AbortableTask ) -def refresh_soap(self, username, layername, http_method='post', metadata_properties_to_refresh=None, access_rights=None): +def refresh_soap(self, username, layername, http_method=common.REQUEST_METHOD_POST, metadata_properties_to_refresh=None, access_rights=None): metadata_properties_to_refresh = metadata_properties_to_refresh or [] if self.is_aborted(): raise AbortedException - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: soap.soap_insert(username, layername, access_rights) else: soap.patch_layer(username, layername, metadata_properties_to_refresh, access_rights) diff --git a/src/layman/layer/rest_workspace_layer.py b/src/layman/layer/rest_workspace_layer.py index b1a219f5a..0652ec01f 100644 --- a/src/layman/layer/rest_workspace_layer.py +++ b/src/layman/layer/rest_workspace_layer.py @@ -112,7 +112,8 @@ def patch(workspace, layername): layer_result = {} if delete_from is not None: - deleted = util.delete_layer(workspace, layername, source=delete_from, http_method='patch') + request_method = request.method.lower() + deleted = util.delete_layer(workspace, layername, source=delete_from, http_method=request_method) if style_file is None: try: style_file = deleted['style']['file'] @@ -127,7 +128,7 @@ def patch(workspace, layername): kwargs.update({ 'crs_id': crs_id, 'ensure_user': False, - 'http_method': 'patch', + 'http_method': request_method, 'metadata_properties_to_refresh': props_to_refresh, }) diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index 8ba9775b9..86f3d821b 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -6,7 +6,7 @@ from layman import LaymanError, patch_mode, util as layman_util from layman.util import call_modules_fn, get_providers_from_source_names, get_internal_sources, \ to_safe_name, url_for -from layman import celery as celery_util +from layman import celery as celery_util, common from layman.common import redis as redis_util, tasks as tasks_util, metadata as metadata_common from layman.common.util import PUBLICATION_NAME_PATTERN, clear_publication_info from . import get_layer_sources, LAYER_TYPE, get_layer_type_def @@ -208,7 +208,7 @@ def delete_layer(workspace, layername, source=None, http_method='delete'): ), 0) end_idx = None if source_idx == 0 else source_idx - 1 sources = sources[:end_idx:-1] - if http_method == 'patch': + if http_method == common.REQUEST_METHOD_PATCH: sources = [ m for m in sources if m.PATCH_MODE == patch_mode.DELETE_IF_DEPENDANT diff --git a/src/layman/map/micka/csw.py b/src/layman/map/micka/csw.py index 7c127c680..43cb94dc2 100644 --- a/src/layman/map/micka/csw.py +++ b/src/layman/map/micka/csw.py @@ -8,7 +8,7 @@ from requests.exceptions import HTTPError, ConnectionError from flask import current_app -from layman import settings, LaymanError +from layman import common, settings, LaymanError from layman.common import language as common_language, empty_method, empty_method_returns_none, bbox as bbox_util from layman.common.filesystem.uuid import get_publication_uuid_file from layman.common.micka import util as common_util @@ -79,7 +79,7 @@ def patch_map(workspace, mapname, metadata_properties_to_refresh=None, actor_nam return None # current_app.logger.info(f"Current element=\n{ET.tostring(el, encoding='unicode', pretty_print=True)}") - _, prop_values = get_template_path_and_values(workspace, mapname, http_method='patch', actor_name=actor_name) + _, prop_values = get_template_path_and_values(workspace, mapname, http_method=common.REQUEST_METHOD_PATCH, actor_name=actor_name) prop_values = { k: v for k, v in prop_values.items() if k in metadata_properties_to_refresh + ['md_date_stamp'] @@ -102,7 +102,7 @@ def patch_map(workspace, mapname, metadata_properties_to_refresh=None, actor_nam def csw_insert(username, mapname, actor_name): - template_path, prop_values = get_template_path_and_values(username, mapname, http_method='post', actor_name=actor_name) + template_path, prop_values = get_template_path_and_values(username, mapname, http_method=common.REQUEST_METHOD_POST, actor_name=actor_name) record = common_util.fill_xml_template_as_pretty_str(template_path, prop_values, METADATA_PROPERTIES) try: muuid = common_util.csw_insert({ @@ -176,7 +176,7 @@ def map_json_to_epsg_codes(map_json): def get_template_path_and_values(username, mapname, http_method=None, actor_name=None): - assert http_method in ['post', 'patch'] + assert http_method in [common.REQUEST_METHOD_POST, common.REQUEST_METHOD_PATCH] uuid_file_path = get_publication_uuid_file(MAP_TYPE, username, mapname) publ_datetime = datetime.fromtimestamp(os.path.getmtime(uuid_file_path)) revision_date = datetime.now() @@ -214,7 +214,7 @@ def get_template_path_and_values(username, mapname, http_method=None, actor_name operates_on=operates_on, md_language=md_language, ) - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: prop_values.pop('revision_date', None) template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'record-template.xml') return template_path, prop_values diff --git a/src/layman/map/micka/tasks.py b/src/layman/map/micka/tasks.py index a3f537483..b397b79e6 100644 --- a/src/layman/map/micka/tasks.py +++ b/src/layman/map/micka/tasks.py @@ -2,7 +2,7 @@ from layman.celery import AbortedException from layman.common import empty_method_returns_true -from layman import celery_app +from layman import celery_app, common from . import csw, soap logger = get_task_logger(__name__) @@ -15,10 +15,10 @@ bind=True, base=celery_app.AbortableTask ) -def refresh_csw(self, username, mapname, http_method='post', metadata_properties_to_refresh=None, actor_name=None): +def refresh_csw(self, username, mapname, http_method=common.REQUEST_METHOD_POST, metadata_properties_to_refresh=None, actor_name=None): if self.is_aborted(): raise AbortedException - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: csw.csw_insert(username, mapname, actor_name=actor_name) else: csw.patch_map(username, mapname, metadata_properties_to_refresh=metadata_properties_to_refresh, actor_name=actor_name) @@ -33,10 +33,10 @@ def refresh_csw(self, username, mapname, http_method='post', metadata_properties bind=True, base=celery_app.AbortableTask ) -def refresh_soap(self, username, mapname, http_method='post', metadata_properties_to_refresh=None, actor_name=None, access_rights=None): +def refresh_soap(self, username, mapname, http_method=common.REQUEST_METHOD_POST, metadata_properties_to_refresh=None, actor_name=None, access_rights=None): if self.is_aborted(): raise AbortedException - if http_method == 'post': + if http_method == common.REQUEST_METHOD_POST: soap.soap_insert(username, mapname, access_rights=access_rights, actor_name=actor_name) else: soap.patch_map(username, mapname, metadata_properties_to_refresh=metadata_properties_to_refresh, actor_name=actor_name, access_rights=access_rights) From e52b63148559df158fe4b308a49a9e3e24921fae Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 7 May 2021 10:52:19 +0200 Subject: [PATCH 05/17] Parameter renaming --- src/layman/common/redis.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/layman/common/redis.py b/src/layman/common/redis.py index fbcacb752..7c7971836 100644 --- a/src/layman/common/redis.py +++ b/src/layman/common/redis.py @@ -62,7 +62,7 @@ def unlock_publication(workspace, publication_type, publication_name): rds.hdel(key, hash) -def solve_locks(workspace, publication_type, publication_name, error_code, method): +def solve_locks(workspace, publication_type, publication_name, error_code, requested_lock): current_lock = get_publication_lock( workspace, publication_type, @@ -70,27 +70,27 @@ def solve_locks(workspace, publication_type, publication_name, error_code, metho ) if current_lock is None: return - if method not in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_DELETE, - common.PUBLICATION_LOCK_CODE_WFST, ]: - raise Exception(f"Unknown method to check: {method}") + if requested_lock not in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_DELETE, + common.PUBLICATION_LOCK_CODE_WFST, ]: + raise Exception(f"Unknown method to check: {requested_lock}") if current_lock not in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_DELETE, common.PUBLICATION_LOCK_CODE_POST, common.PUBLICATION_LOCK_CODE_WFST, ]: raise Exception(f"Unknown current lock: {current_lock}") if current_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: - if method in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: + if requested_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: raise LaymanError(error_code) elif current_lock in [common.PUBLICATION_LOCK_CODE_DELETE, ]: - if method in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: + if requested_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: raise LaymanError(error_code) - if method not in [common.PUBLICATION_LOCK_CODE_DELETE, ]: - if (current_lock, method) == (common.PUBLICATION_LOCK_CODE_WFST, common.PUBLICATION_LOCK_CODE_WFST): + if requested_lock not in [common.PUBLICATION_LOCK_CODE_DELETE, ]: + if (current_lock, requested_lock) == (common.PUBLICATION_LOCK_CODE_WFST, common.PUBLICATION_LOCK_CODE_WFST): chain_info = celery_util.get_publication_chain_info(workspace, publication_type, publication_name) celery_util.abort_chain(chain_info) else: - assert current_lock not in [common.PUBLICATION_LOCK_CODE_WFST, ] and method not in [ + assert current_lock not in [common.PUBLICATION_LOCK_CODE_WFST, ] and requested_lock not in [ common.PUBLICATION_LOCK_CODE_WFST, ],\ - f'current_lock={current_lock}, method={method},' \ + f'current_lock={current_lock}, method={requested_lock},' \ f'workspace, publication_type, publication_name={(workspace, publication_type, publication_name)}' From 0a6165d9b3b1144dc76f74e796e773253a48b8c3 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 12:42:39 +0200 Subject: [PATCH 06/17] Set timeout for src/layman/geoserver_proxy_test.py::test_missing_attribute to 60 --- src/layman/geoserver_proxy_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/layman/geoserver_proxy_test.py b/src/layman/geoserver_proxy_test.py index 792b10ea3..9274f56b0 100644 --- a/src/layman/geoserver_proxy_test.py +++ b/src/layman/geoserver_proxy_test.py @@ -159,6 +159,7 @@ def test_wms_ows_proxy(service_endpoint): client_util.delete_workspace_layer(username, layername, headers=authn_headers) +@pytest.mark.timeout(60) @pytest.mark.usefixtures('ensure_layman', 'liferay_mock') @pytest.mark.parametrize('style_file', [ None, From 2721d8456dc2f2ea089d3708926a86b10ea22dc6 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 15:12:55 +0200 Subject: [PATCH 07/17] Solve empty value in cache for wms and wfs --- src/layman/layer/geoserver/wfs.py | 2 +- src/layman/layer/geoserver/wms.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layman/layer/geoserver/wfs.py b/src/layman/layer/geoserver/wfs.py index 9d157b834..ede3b5906 100644 --- a/src/layman/layer/geoserver/wfs.py +++ b/src/layman/layer/geoserver/wfs.py @@ -61,7 +61,7 @@ def get_wfs_direct(username): from .util import wfs_direct key = get_flask_proxy_key(username) redis_obj = settings.LAYMAN_REDIS.hgetall(key) - string_value = redis_obj['value'] if redis_obj is not None else None + string_value = redis_obj['value'] if redis_obj else None return wfs_direct(ows_url, xml=string_value, headers=headers) diff --git a/src/layman/layer/geoserver/wms.py b/src/layman/layer/geoserver/wms.py index 58aae8497..e7d996bfb 100644 --- a/src/layman/layer/geoserver/wms.py +++ b/src/layman/layer/geoserver/wms.py @@ -68,7 +68,7 @@ def get_wms_direct(username): from .util import wms_direct key = get_flask_proxy_key(username) redis_obj = settings.LAYMAN_REDIS.hgetall(key) - string_value = redis_obj['value'] if redis_obj is not None else None + string_value = redis_obj['value'] if redis_obj else None return wms_direct(ows_url, xml=string_value, headers=headers) From b718cb14569229c880310cefad8bcadb5eda564b Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 6 May 2021 15:13:43 +0200 Subject: [PATCH 08/17] Create queue of tasks to run after current chain ends for each publication --- src/layman/celery.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/layman/celery.py b/src/layman/celery.py index 585670a4f..b23b339cf 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -8,6 +8,7 @@ REDIS_CURRENT_TASK_NAMES = f"{__name__}:CURRENT_TASK_NAMES" PUBLICATION_CHAIN_INFOS = f'{__name__}:PUBLICATION_TASK_INFOS' LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION = f'{__name__}:TASK_ID_TO_PUBLICATION' +RUN_AFTER_CHAIN = f'{__name__}:RUN_AFTER_CHAIN' def task_prerun(workspace, _publication_type, publication_name, _task_id, task_name): @@ -40,6 +41,38 @@ def _get_task_hash(task_name, workspace, publication_name): return f"{task_name}:{workspace}:{publication_name}" +def push_step_to_run_after_chain(workspace, publication_type, publication_name, step_code, ): + rds = settings.LAYMAN_REDIS + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION + hash = _get_publication_hash(workspace, publication_type, publication_name) + val = rds.hget(key, hash) + queue = json.loads(val) if val is not None else list() + if step_code not in queue: + queue.append(step_code) + rds.hset(key, hash, json.dumps(queue)) + + +def pop_step_to_run_after_chain(workspace, publication_type, publication_name, ): + rds = settings.LAYMAN_REDIS + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION + hash = _get_publication_hash(workspace, publication_type, publication_name) + val = rds.hget(key, hash) + result = None + if val: + queue = json.loads(val) + if len(queue) > 0: + result = queue.pop(0) + rds.hset(key, hash, json.dumps(queue)) + return result + + +def clear_steps_to_run_after_chain(workspace, publication_type, publication_name, ): + rds = settings.LAYMAN_REDIS + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION + hash = _get_publication_hash(workspace, publication_type, publication_name) + rds.hdel(key, hash) + + def finnish_publication_task(task_id): rds = settings.LAYMAN_REDIS key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION From 55e99889d82528418f7ecdf73262e8709e3db192 Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 7 May 2021 11:29:16 +0200 Subject: [PATCH 09/17] Create celery.abort_publication_chain and use it --- src/layman/celery.py | 6 ++++++ src/layman/common/redis.py | 3 +-- src/layman/layer/util.py | 3 +-- src/layman/map/util.py | 3 +-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/layman/celery.py b/src/layman/celery.py index b23b339cf..bfeec7f52 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -179,6 +179,12 @@ def abort_chain(chain_info): finnish_publication_task(chain_info['last'].task_id) +def abort_publication_chain(workspace, publication_type, publication_name): + chain_info = get_publication_chain_info(workspace, publication_type, publication_name) + abort_chain(chain_info) + clear_steps_to_run_after_chain(workspace, publication_type, publication_name) + + def abort_task_chain(results_by_order, results_by_name=None): results_by_name = results_by_name or {} task_results = [r for r in results_by_order if not r.ready()] diff --git a/src/layman/common/redis.py b/src/layman/common/redis.py index 7c7971836..e66defde8 100644 --- a/src/layman/common/redis.py +++ b/src/layman/common/redis.py @@ -85,8 +85,7 @@ def solve_locks(workspace, publication_type, publication_name, error_code, reque raise LaymanError(error_code) if requested_lock not in [common.PUBLICATION_LOCK_CODE_DELETE, ]: if (current_lock, requested_lock) == (common.PUBLICATION_LOCK_CODE_WFST, common.PUBLICATION_LOCK_CODE_WFST): - chain_info = celery_util.get_publication_chain_info(workspace, publication_type, publication_name) - celery_util.abort_chain(chain_info) + celery_util.abort_publication_chain(workspace, publication_type, publication_name) else: assert current_lock not in [common.PUBLICATION_LOCK_CODE_WFST, ] and requested_lock not in [ common.PUBLICATION_LOCK_CODE_WFST, ],\ diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index 86f3d821b..8290ef0a6 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -230,8 +230,7 @@ def _get_layer_chain(username, layername): def abort_layer_chain(username, layername): - chain_info = _get_layer_chain(username, layername) - celery_util.abort_chain(chain_info) + celery_util.abort_publication_chain(username, LAYER_TYPE, layername) def is_layer_chain_ready(username, layername): diff --git a/src/layman/map/util.py b/src/layman/map/util.py index ad1d0107a..0a479bcb1 100644 --- a/src/layman/map/util.py +++ b/src/layman/map/util.py @@ -221,8 +221,7 @@ def _get_map_chain(username, mapname): def abort_map_chain(username, mapname): - chain_info = _get_map_chain(username, mapname) - celery_util.abort_chain(chain_info) + celery_util.abort_publication_chain(username, MAP_TYPE, mapname) def is_map_chain_ready(username, mapname): From 485219164f76133e57e2860e4cfb70ab851a34b3 Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 7 May 2021 11:42:03 +0200 Subject: [PATCH 10/17] Solve concurrency of WFS-T and other locks --- src/layman/celery.py | 7 +++++ src/layman/common/redis.py | 15 +++++---- src/layman/geoserver_proxy.py | 13 ++++++-- src/layman/requests_concurrency_test.py | 41 +++++++++++++++++++++++++ src/layman/util.py | 6 ++++ 5 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 src/layman/requests_concurrency_test.py diff --git a/src/layman/celery.py b/src/layman/celery.py index bfeec7f52..d147c369d 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -1,4 +1,5 @@ import json +import importlib from flask import current_app from celery.contrib.abortable import AbortableAsyncResult @@ -30,6 +31,12 @@ def task_postrun(workspace, publication_type, publication_name, task_id, task_na hash = task_id if rds.hexists(key, hash): finnish_publication_task(task_id) + next_task = pop_step_to_run_after_chain(workspace, publication_type, publication_name) + if next_task: + module_name, method_name = next_task.split('::') + module = importlib.import_module(module_name) + method = getattr(module, method_name) + method(workspace, publication_type, publication_name) elif task_state == 'FAILURE': chain_info = get_publication_chain_info_dict(workspace, publication_type, publication_name) if chain_info is not None: diff --git a/src/layman/common/redis.py b/src/layman/common/redis.py index e66defde8..df40419d6 100644 --- a/src/layman/common/redis.py +++ b/src/layman/common/redis.py @@ -81,16 +81,15 @@ def solve_locks(workspace, publication_type, publication_name, error_code, reque if requested_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: raise LaymanError(error_code) elif current_lock in [common.PUBLICATION_LOCK_CODE_DELETE, ]: - if requested_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, ]: + if requested_lock in [common.PUBLICATION_LOCK_CODE_PATCH, common.PUBLICATION_LOCK_CODE_POST, common.PUBLICATION_LOCK_CODE_WFST, ]: raise LaymanError(error_code) if requested_lock not in [common.PUBLICATION_LOCK_CODE_DELETE, ]: - if (current_lock, requested_lock) == (common.PUBLICATION_LOCK_CODE_WFST, common.PUBLICATION_LOCK_CODE_WFST): - celery_util.abort_publication_chain(workspace, publication_type, publication_name) - else: - assert current_lock not in [common.PUBLICATION_LOCK_CODE_WFST, ] and requested_lock not in [ - common.PUBLICATION_LOCK_CODE_WFST, ],\ - f'current_lock={current_lock}, method={requested_lock},' \ - f'workspace, publication_type, publication_name={(workspace, publication_type, publication_name)}' + if requested_lock == common.PUBLICATION_LOCK_CODE_WFST: + raise LaymanError(19, private_data={'can_run_later': True}) + if current_lock == common.PUBLICATION_LOCK_CODE_WFST and requested_lock in [common.REQUEST_METHOD_PATCH, common.REQUEST_METHOD_POST, ]: + chain_info = celery_util.get_publication_chain_info(workspace, publication_type, publication_name) + celery_util.abort_chain(chain_info) + celery_util.push_step_to_run_after_chain(workspace, publication_type, publication_name, 'layman.util::patch_after_wfst') def _get_publication_hash(workspace, publication_type, publication_name): diff --git a/src/layman/geoserver_proxy.py b/src/layman/geoserver_proxy.py index d95c74b07..c94973dc1 100644 --- a/src/layman/geoserver_proxy.py +++ b/src/layman/geoserver_proxy.py @@ -7,7 +7,7 @@ from flask import Blueprint, g, current_app as app, request, Response from geoserver.util import reset as gs_reset -from layman import authn, authz, common, settings +from layman import authn, authz, common, settings, LaymanError, celery as celery_util from layman.authn import authenticate, is_user_with_name from layman.common import redis from layman.layer import db, LAYER_TYPE, util as layer_util @@ -197,8 +197,15 @@ def proxy(subpath): for workspace, layername in wfs_t_layers: if authz.can_i_edit(LAYER_TYPE, workspace, layername): - redis.create_lock(workspace, LAYER_TYPE, layername, 19, common.PUBLICATION_LOCK_CODE_WFST) - patch_after_wfst(workspace, layername) + try: + redis.create_lock(workspace, LAYER_TYPE, layername, 19, common.PUBLICATION_LOCK_CODE_WFST) + patch_after_wfst(workspace, layername) + except LaymanError as exc: + if exc.code == 19 and exc.private_data.get('can_run_later', False): + celery_util.push_step_to_run_after_chain(workspace, LAYER_TYPE, layername, + 'layman.util::patch_after_wfst') + else: + raise exc excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] headers = {key: value for (key, value) in response.headers.items() if key.lower() not in excluded_headers} diff --git a/src/layman/requests_concurrency_test.py b/src/layman/requests_concurrency_test.py new file mode 100644 index 000000000..2d3cf6909 --- /dev/null +++ b/src/layman/requests_concurrency_test.py @@ -0,0 +1,41 @@ +from test import process_client +from test.data import wfs as data_wfs +import requests +import pytest + +from layman import settings +from layman.common import empty_method_returns_true + + +@pytest.mark.usefixtures('ensure_layman') +def test_wfst_concurrency(): + workspace = 'test_wfst_concurrency_workspace' + layer = 'test_wfst_concurrency_layer' + + data_xml = data_wfs.get_wfs20_insert_points(workspace, layer, ) + rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{workspace}/wfs?request=Transaction" + headers = { + 'Accept': 'text/xml', + 'Content-type': 'text/xml', + } + + process_client.publish_workspace_layer(workspace, layer,) + + r = requests.post(rest_url, + data=data_xml, + headers=headers) + assert r.status_code == 200, r.text + + process_client.patch_workspace_layer(workspace, layer, title='New title', check_response_fn=empty_method_returns_true) + + r = requests.post(rest_url, + data=data_xml, + headers=headers) + assert r.status_code == 200, r.text + + r = requests.post(rest_url, + data=data_xml, + headers=headers) + assert r.status_code == 200, r.text + + process_client.delete_workspace_layer(workspace, layer, ) diff --git a/src/layman/util.py b/src/layman/util.py index d48824cfe..19678b059 100644 --- a/src/layman/util.py +++ b/src/layman/util.py @@ -387,3 +387,9 @@ def delete_publications(user, for (name, info) in whole_infos.items() ] return jsonify(infos) + + +def patch_after_wfst(workspace, publication_type, layername, ): + from layman.layer import LAYER_TYPE, util as layer_util + if publication_type == LAYER_TYPE: + layer_util.patch_after_wfst(workspace, layername) From 1de5ad2fec00cac9994b0b295227ee14da72887e Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 7 May 2021 11:52:10 +0200 Subject: [PATCH 11/17] Remove redundant sleeps --- src/layman/geoserver_proxy_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/layman/geoserver_proxy_test.py b/src/layman/geoserver_proxy_test.py index 9274f56b0..f07d2b40b 100644 --- a/src/layman/geoserver_proxy_test.py +++ b/src/layman/geoserver_proxy_test.py @@ -297,8 +297,6 @@ def wfs_post(workspace, attr_names_list, data_xml): data_xml = data_wfs.get_wfs11_insert_polygon_new_attr(username, layername, attr_names10) wfs_post(username, [(layername, attr_names10)], data_xml) - time.sleep(5) - client_util.delete_workspace_layer(username, layername, headers=headers) client_util.delete_workspace_layer(username, layername2, headers=headers) @@ -368,8 +366,6 @@ def do_test(wfs_query, attribute_names): data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername1, attr_names) do_test(data_xml, attr_names) - time.sleep(5) - client_util.delete_workspace_layer(username, layername1, headers=headers1) From 9e264d39f09f0caad29b0468d994adbadd020f33 Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 09:42:15 +0200 Subject: [PATCH 12/17] Rename redis caches keys --- src/layman/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layman/celery.py b/src/layman/celery.py index d147c369d..ab5a165a3 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -7,8 +7,8 @@ from layman.common import redis as redis_util REDIS_CURRENT_TASK_NAMES = f"{__name__}:CURRENT_TASK_NAMES" -PUBLICATION_CHAIN_INFOS = f'{__name__}:PUBLICATION_TASK_INFOS' -LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION = f'{__name__}:TASK_ID_TO_PUBLICATION' +PUBLICATION_CHAIN_INFOS = f'{__name__}:PUBLICATION_CHAIN_INFOS' +LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION = f'{__name__}:LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION' RUN_AFTER_CHAIN = f'{__name__}:RUN_AFTER_CHAIN' From 2aec8953574a6bb98bcda2184d33a1d541e80b72 Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 09:43:37 +0200 Subject: [PATCH 13/17] Rename finnish_publication_task to finnish_publication_chain --- src/layman/celery.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/layman/celery.py b/src/layman/celery.py index ab5a165a3..caf1d75bb 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -30,7 +30,7 @@ def task_postrun(workspace, publication_type, publication_name, task_id, task_na key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION hash = task_id if rds.hexists(key, hash): - finnish_publication_task(task_id) + finnish_publication_chain(task_id) next_task = pop_step_to_run_after_chain(workspace, publication_type, publication_name) if next_task: module_name, method_name = next_task.split('::') @@ -41,7 +41,7 @@ def task_postrun(workspace, publication_type, publication_name, task_id, task_na chain_info = get_publication_chain_info_dict(workspace, publication_type, publication_name) if chain_info is not None: last_task_id = chain_info['last'] - finnish_publication_task(last_task_id) + finnish_publication_chain(last_task_id) def _get_task_hash(task_name, workspace, publication_name): @@ -80,10 +80,10 @@ def clear_steps_to_run_after_chain(workspace, publication_type, publication_name rds.hdel(key, hash) -def finnish_publication_task(task_id): +def finnish_publication_chain(last_task_id_in_chain): rds = settings.LAYMAN_REDIS key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION - hash = task_id + hash = last_task_id_in_chain publ_hash = rds.hget(key, hash) if publ_hash is None: return @@ -183,7 +183,7 @@ def abort_chain(chain_info): return abort_task_chain(chain_info['by_order'], chain_info['by_name']) - finnish_publication_task(chain_info['last'].task_id) + finnish_publication_chain(chain_info['last'].task_id) def abort_publication_chain(workspace, publication_type, publication_name): From d8233aa7f7ba0a6d7d8808a8b75c04fee543bc94 Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 14:04:08 +0200 Subject: [PATCH 14/17] Move assertion utils to separate file due to circular dependency with process_client --- src/layman/common/output_srs_list_test.py | 4 +- src/layman/geoserver_proxy_test.py | 32 ++------ src/layman/layer/geoserver/geoserver_test.py | 12 +-- src/layman/layer/geoserver/sld_test.py | 4 +- src/layman/layer/prime_db_schema/bbox_test.py | 6 +- src/layman/map/prime_db_schema/bbox_test.py | 6 +- src/layman/upgrade/upgrade_v1_10_test.py | 6 +- src/layman/upgrade/upgrade_v1_12_test.py | 4 +- test/assert_util.py | 80 +++++++++++++++++++ test/util.py | 54 ------------- 10 files changed, 106 insertions(+), 102 deletions(-) create mode 100644 test/assert_util.py diff --git a/src/layman/common/output_srs_list_test.py b/src/layman/common/output_srs_list_test.py index c336b963b..0d0db6050 100644 --- a/src/layman/common/output_srs_list_test.py +++ b/src/layman/common/output_srs_list_test.py @@ -1,5 +1,5 @@ import math -from test import process, process_client, geoserver_client, util +from test import process, process_client, geoserver_client, assert_util import pytest from layman import settings, app @@ -171,4 +171,4 @@ def test_spatial_precision_wms(ensure_layer, epsg_code, extent, img_size, style_ num_circles = 5 pixel_diff_limit = circle_perimeter * num_circles * diff_line_width - util.assert_same_images(url, obtained_file, expected_file, pixel_diff_limit) + assert_util.assert_same_images(url, obtained_file, expected_file, pixel_diff_limit) diff --git a/src/layman/geoserver_proxy_test.py b/src/layman/geoserver_proxy_test.py index f07d2b40b..53b01a3eb 100644 --- a/src/layman/geoserver_proxy_test.py +++ b/src/layman/geoserver_proxy_test.py @@ -1,5 +1,6 @@ import time -from test import process_client as client_util, geoserver_client, util as test_util + +from test import process_client as client_util, geoserver_client, util as test_util, assert_util from test.process_client import get_authz_headers from test.data import wfs as data_wfs, SMALL_LAYER_BBOX import requests @@ -7,8 +8,7 @@ import pytest from geoserver.util import get_layer_thumbnail, get_square_bbox -from layman import app, settings, util as layman_util -from layman.common import bbox as bbox_util +from layman import app, settings from layman.layer import db, util as layer_util from layman.layer.filesystem import thumbnail from layman.layer.geoserver import wfs as geoserver_wfs @@ -369,28 +369,6 @@ def do_test(wfs_query, attribute_names): client_util.delete_workspace_layer(username, layername1, headers=headers1) -def assert_all_sources_bbox(workspace, layer, expected_bbox): - with app.app_context(): - bbox = tuple(layman_util.get_publication_info(workspace, client_util.LAYER_TYPE, layer, - context={'key': ['bounding_box']})['bounding_box']) - test_util.assert_same_bboxes(expected_bbox, bbox, 0) - test_util.assert_wfs_bbox(workspace, layer, expected_bbox) - test_util.assert_wms_bbox(workspace, layer, expected_bbox) - - with app.app_context(): - expected_bbox_4326 = bbox_util.transform(expected_bbox, 3857, 4326, ) - md_comparison = client_util.get_workspace_layer_metadata_comparison(workspace, layer) - csw_prefix = settings.CSW_PROXY_URL - csw_src_key = client_util.get_source_key_from_metadata_comparison(md_comparison, csw_prefix) - assert csw_src_key is not None - prop_key = 'extent' - md_props = md_comparison['metadata_properties'] - assert md_props[prop_key]['equal'] is True - assert md_props[prop_key]['equal_or_null'] is True - csw_bbox_4326 = tuple(md_props[prop_key]['values'][csw_src_key]) - test_util.assert_same_bboxes(expected_bbox_4326, csw_bbox_4326, 0.001) - - @pytest.mark.parametrize('style_file, thumbnail_style_postfix', [ (None, '_sld'), ('sample/style/small_layer.qml', '_qml'), @@ -402,7 +380,7 @@ def test_wfs_bbox(style_file, thumbnail_style_postfix): client_util.publish_workspace_layer(workspace, layer, style_file=style_file, ) - assert_all_sources_bbox(workspace, layer, SMALL_LAYER_BBOX) + assert_util.assert_all_sources_bbox(workspace, layer, SMALL_LAYER_BBOX) rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{workspace}/wfs?request=Transaction" headers = { @@ -427,7 +405,7 @@ def test_wfs_bbox(style_file, thumbnail_style_postfix): # until there is way to check end of asynchronous task after WFS-T time.sleep(5) - assert_all_sources_bbox(workspace, layer, exp_bbox) + assert_util.assert_all_sources_bbox(workspace, layer, exp_bbox) expected_thumbnail_path = f'/code/sample/style/{layer}{thumbnail_style_postfix}{thumbnail_bbox_postfix}.png' with app.app_context(): diff --git a/src/layman/layer/geoserver/geoserver_test.py b/src/layman/layer/geoserver/geoserver_test.py index 6895bb2fa..8853bfa89 100644 --- a/src/layman/layer/geoserver/geoserver_test.py +++ b/src/layman/layer/geoserver/geoserver_test.py @@ -1,4 +1,4 @@ -from test import process_client, util as test_util, data as test_data +from test import process_client, assert_util, data as test_data import pytest from layman import app, settings @@ -46,8 +46,8 @@ def test_geoserver_bbox(): process_client.publish_workspace_layer(workspace, layer, style_file='sample/style/small_layer.qml') - test_util.assert_wfs_bbox(workspace, layer, expected_bbox_1) - test_util.assert_wms_bbox(workspace, layer, expected_bbox_1) + assert_util.assert_wfs_bbox(workspace, layer, expected_bbox_1) + assert_util.assert_wms_bbox(workspace, layer, expected_bbox_1) # test WFS for bbox, expected_bbox in expected_bboxes: @@ -61,7 +61,7 @@ def test_geoserver_bbox(): ensure_user=False, access_rights=None, ) - test_util.assert_wfs_bbox(workspace, layer, expected_bbox) + assert_util.assert_wfs_bbox(workspace, layer, expected_bbox) # test WMS for bbox, expected_bbox in expected_bboxes: @@ -74,7 +74,7 @@ def test_geoserver_bbox(): ensure_user=False, access_rights=None, ) - test_util.assert_wms_bbox(workspace, layer, expected_bbox) + assert_util.assert_wms_bbox(workspace, layer, expected_bbox) # test cascade WMS from QGIS for bbox, expected_bbox in expected_bboxes: @@ -88,6 +88,6 @@ def test_geoserver_bbox(): ensure_user=False, access_rights=None, ) - test_util.assert_wms_bbox(workspace, layer, expected_bbox) + assert_util.assert_wms_bbox(workspace, layer, expected_bbox) process_client.delete_workspace_layer(workspace, layer) diff --git a/src/layman/layer/geoserver/sld_test.py b/src/layman/layer/geoserver/sld_test.py index 4c518bec7..6671ae624 100644 --- a/src/layman/layer/geoserver/sld_test.py +++ b/src/layman/layer/geoserver/sld_test.py @@ -1,5 +1,5 @@ from urllib.parse import urljoin -from test import process_client, util +from test import process_client, assert_util import requests import pytest @@ -53,7 +53,7 @@ def test_sld_style_applied_in_wms(): url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{workspace}_wms/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image/png&TRANSPARENT=true&STYLES=&LAYERS={workspace}:{layer}&SRS=EPSG:3857&WIDTH=768&HEIGHT=752&BBOX=-30022616.05686392,-30569903.32873383,30022616.05686392,28224386.44929134" - util.assert_same_images(url, obtained_file, expected_file, 2000) + assert_util.assert_same_images(url, obtained_file, expected_file, 2000) process_client.delete_workspace_layer(workspace, layer) diff --git a/src/layman/layer/prime_db_schema/bbox_test.py b/src/layman/layer/prime_db_schema/bbox_test.py index 28e208287..4a87c1f16 100644 --- a/src/layman/layer/prime_db_schema/bbox_test.py +++ b/src/layman/layer/prime_db_schema/bbox_test.py @@ -1,4 +1,4 @@ -from test import process_client, util as test_util, data as test_data +from test import process_client, assert_util, data as test_data import pytest from layman import app from .. import util @@ -13,12 +13,12 @@ def test_bbox(): with app.app_context(): info = util.get_layer_info(workspace, layer) - test_util.assert_same_bboxes(info['bounding_box'], test_data.SMALL_LAYER_BBOX, 0.00001) + assert_util.assert_same_bboxes(info['bounding_box'], test_data.SMALL_LAYER_BBOX, 0.00001) process_client.patch_workspace_layer(workspace, layer, file_paths=['test/data/bbox/layer_3_3-5_5.geojson', ]) with app.app_context(): info = util.get_layer_info(workspace, layer) - test_util.assert_same_bboxes(info['bounding_box'], [3000, 3000, 5000, 5000], 0.1) + assert_util.assert_same_bboxes(info['bounding_box'], [3000, 3000, 5000, 5000], 0.1) process_client.delete_workspace_layer(workspace, layer) diff --git a/src/layman/map/prime_db_schema/bbox_test.py b/src/layman/map/prime_db_schema/bbox_test.py index a77016ba6..88d28c4a4 100644 --- a/src/layman/map/prime_db_schema/bbox_test.py +++ b/src/layman/map/prime_db_schema/bbox_test.py @@ -1,4 +1,4 @@ -from test import process_client, util as test_util, data as test_data +from test import process_client, assert_util, data as test_data import pytest from layman import app from .. import util @@ -13,12 +13,12 @@ def test_bbox(): with app.app_context(): info = util.get_map_info(workspace, map) - test_util.assert_same_bboxes(info['bounding_box'], test_data.SMALL_MAP_BBOX, 0.00001) + assert_util.assert_same_bboxes(info['bounding_box'], test_data.SMALL_MAP_BBOX, 0.00001) process_client.patch_workspace_map(workspace, map, file_paths=['test/data/bbox/map_3_3-5_5.json', ]) with app.app_context(): info = util.get_map_info(workspace, map) - test_util.assert_same_bboxes(info['bounding_box'], [3000, 3000, 5000, 5000], 0.1) + assert_util.assert_same_bboxes(info['bounding_box'], [3000, 3000, 5000, 5000], 0.1) process_client.delete_workspace_map(workspace, map) diff --git a/src/layman/upgrade/upgrade_v1_10_test.py b/src/layman/upgrade/upgrade_v1_10_test.py index b1ae98477..66ef0a4d2 100644 --- a/src/layman/upgrade/upgrade_v1_10_test.py +++ b/src/layman/upgrade/upgrade_v1_10_test.py @@ -3,7 +3,7 @@ import shutil import os from collections import namedtuple -from test import process_client, util +from test import process_client, util, assert_util from test.util import url_for import pytest @@ -142,7 +142,7 @@ def test_migrate_layers_to_wms_workspace(ensure_layer): f"BBOX=-30022616.05686392,-30569903.32873383,30022616.05686392,28224386.44929134" obtained_file = 'tmp/artifacts/test_migrate_layers_to_wms_workspace_before_migration.png' - util.assert_same_images(old_wms_url, obtained_file, expected_file, 2000) + assert_util.assert_same_images(old_wms_url, obtained_file, expected_file, 2000) with app.app_context(): upgrade_v1_10.migrate_layers_to_wms_workspace(workspace) @@ -170,7 +170,7 @@ def test_migrate_layers_to_wms_workspace(ensure_layer): f"LAYERS={wms_workspace}:{layer}&SRS=EPSG:3857&WIDTH=768&HEIGHT=752&" \ f"BBOX=-30022616.05686392,-30569903.32873383,30022616.05686392,28224386.44929134" obtained_file2 = 'tmp/artifacts/test_migrate_layers_to_wms_workspace_after_migration.png' - util.assert_same_images(new_wms_url, obtained_file2, expected_file, 2000) + assert_util.assert_same_images(new_wms_url, obtained_file2, expected_file, 2000) process_client.delete_workspace_layer(workspace, layer) diff --git a/src/layman/upgrade/upgrade_v1_12_test.py b/src/layman/upgrade/upgrade_v1_12_test.py index 9019ceea9..a0dcb5ee6 100644 --- a/src/layman/upgrade/upgrade_v1_12_test.py +++ b/src/layman/upgrade/upgrade_v1_12_test.py @@ -1,5 +1,5 @@ import datetime -from test import process_client, util as test_util, data as test_data +from test import process_client, assert_util, data as test_data import pytest from db import util as db_util @@ -188,6 +188,6 @@ def test_adjust_prime_db_schema_for_bbox_search(): results = db_util.run_query(query, (workspace, publication_type, publication)) assert len(results) == 1 and len(results[0]) == 4, results bbox = results[0] - test_util.assert_same_bboxes(bbox, expected_bbox, 0.000001) + assert_util.assert_same_bboxes(bbox, expected_bbox, 0.000001) process_client.delete_workspace_layer(workspace, layer) diff --git a/test/assert_util.py b/test/assert_util.py new file mode 100644 index 000000000..faf3188d7 --- /dev/null +++ b/test/assert_util.py @@ -0,0 +1,80 @@ +import os +import pathlib +import requests + +from layman import app, util as layman_util, settings +from layman.common import bbox as bbox_util +from layman.layer.geoserver import wfs, wms +from .process_client import LAYER_TYPE, get_workspace_layer_metadata_comparison, get_source_key_from_metadata_comparison +from .util import compare_images + + +def assert_same_images(img_url, tmp_file_path, expected_file_path, diff_threshold): + r = requests.get(img_url, + timeout=5, + ) + r.raise_for_status() + pathlib.Path(os.path.dirname(tmp_file_path)).mkdir(parents=True, exist_ok=True) + with open(tmp_file_path, 'wb') as f: + for chunk in r: + f.write(chunk) + + diffs = compare_images(expected_file_path, tmp_file_path) + + assert diffs < diff_threshold, f"{diffs} >= {diff_threshold}" + + os.remove(tmp_file_path) + + +def assert_same_bboxes(bbox1, bbox2, precision): + assert len(bbox1) == 4, (bbox1, len(bbox1)) + assert len(bbox2) == 4, (bbox2, len(bbox2)) + for i in range(0, 3): + assert abs(bbox2[i] - bbox1[i]) <= precision, (bbox1, bbox2, precision, i) + + +def assert_wfs_bbox(workspace, layer, expected_bbox): + wfs_layer = f"{workspace}:{layer}" + with app.app_context(): + wfs_get_capabilities = wfs.get_wfs_proxy(workspace) + wfs_bbox_4326 = wfs_get_capabilities.contents[wfs_layer].boundingBoxWGS84 + with app.app_context(): + wfs_bbox_3857 = bbox_util.transform(wfs_bbox_4326, 4326, 3857, ) + assert_same_bboxes(expected_bbox, wfs_bbox_3857, 0.00001) + + +def assert_wms_bbox(workspace, layer, expected_bbox): + with app.app_context(): + wms_get_capabilities = wms.get_wms_proxy(workspace) + wms_layer = wms_get_capabilities.contents[layer] + bbox_3857 = next(bbox[:4] for bbox in wms_layer.crs_list if bbox[4] == 'EPSG:3857') + assert_same_bboxes(expected_bbox, bbox_3857, 0.00001) + + with app.app_context(): + expected_bbox_4326 = bbox_util.transform(expected_bbox, 3857, 4326, ) + wgs84_bboxes = [bbox[:4] for bbox in wms_layer.crs_list if bbox[4] in ['EPSG:4326', 'CRS:84']] + wgs84_bboxes.append(wms_layer.boundingBoxWGS84) + for wgs84_bbox in wgs84_bboxes: + assert_same_bboxes(expected_bbox_4326, wgs84_bbox, 0.00001) + + +def assert_all_sources_bbox(workspace, layer, expected_bbox): + with app.app_context(): + bbox = tuple(layman_util.get_publication_info(workspace, LAYER_TYPE, layer, + context={'key': ['bounding_box']})['bounding_box']) + assert_same_bboxes(expected_bbox, bbox, 0) + assert_wfs_bbox(workspace, layer, expected_bbox) + assert_wms_bbox(workspace, layer, expected_bbox) + + with app.app_context(): + expected_bbox_4326 = bbox_util.transform(expected_bbox, 3857, 4326, ) + md_comparison = get_workspace_layer_metadata_comparison(workspace, layer) + csw_prefix = settings.CSW_PROXY_URL + csw_src_key = get_source_key_from_metadata_comparison(md_comparison, csw_prefix) + assert csw_src_key is not None + prop_key = 'extent' + md_props = md_comparison['metadata_properties'] + assert md_props[prop_key]['equal'] is True, md_props[prop_key] + assert md_props[prop_key]['equal_or_null'] is True, md_props[prop_key] + csw_bbox_4326 = tuple(md_props[prop_key]['values'][csw_src_key]) + assert_same_bboxes(expected_bbox_4326, csw_bbox_4326, 0.001) diff --git a/test/util.py b/test/util.py index 01921df6d..83694fc68 100644 --- a/test/util.py +++ b/test/util.py @@ -1,13 +1,8 @@ -import os -import pathlib import time import requests from requests.exceptions import ConnectionError from PIL import Image, ImageChops -from layman import app -from layman.common import bbox as bbox_util -from layman.layer.geoserver import wfs, wms from layman.util import url_for as layman_url_for @@ -53,52 +48,3 @@ def compare_images(image1, image2): diffs += 1 return diffs - - -def assert_same_images(img_url, tmp_file_path, expected_file_path, diff_threshold): - r = requests.get(img_url, - timeout=5, - ) - r.raise_for_status() - pathlib.Path(os.path.dirname(tmp_file_path)).mkdir(parents=True, exist_ok=True) - with open(tmp_file_path, 'wb') as f: - for chunk in r: - f.write(chunk) - - diffs = compare_images(expected_file_path, tmp_file_path) - - assert diffs < diff_threshold, f"{diffs} >= {diff_threshold}" - - os.remove(tmp_file_path) - - -def assert_same_bboxes(bbox1, bbox2, precision): - assert len(bbox1) == 4, (bbox1, len(bbox1)) - assert len(bbox2) == 4, (bbox2, len(bbox2)) - for i in range(0, 3): - assert abs(bbox2[i] - bbox1[i]) <= precision, (bbox1, bbox2, precision, i) - - -def assert_wfs_bbox(workspace, layer, expected_bbox): - wfs_layer = f"{workspace}:{layer}" - with app.app_context(): - wfs_get_capabilities = wfs.get_wfs_proxy(workspace) - wfs_bbox_4326 = wfs_get_capabilities.contents[wfs_layer].boundingBoxWGS84 - with app.app_context(): - wfs_bbox_3857 = bbox_util.transform(wfs_bbox_4326, 4326, 3857, ) - assert_same_bboxes(expected_bbox, wfs_bbox_3857, 0.00001) - - -def assert_wms_bbox(workspace, layer, expected_bbox): - with app.app_context(): - wms_get_capabilities = wms.get_wms_proxy(workspace) - wms_layer = wms_get_capabilities.contents[layer] - bbox_3857 = next(bbox[:4] for bbox in wms_layer.crs_list if bbox[4] == 'EPSG:3857') - assert_same_bboxes(expected_bbox, bbox_3857, 0.00001) - - with app.app_context(): - expected_bbox_4326 = bbox_util.transform(expected_bbox, 3857, 4326, ) - wgs84_bboxes = [bbox[:4] for bbox in wms_layer.crs_list if bbox[4] in ['EPSG:4326', 'CRS:84']] - wgs84_bboxes.append(wms_layer.boundingBoxWGS84) - for wgs84_bbox in wgs84_bboxes: - assert_same_bboxes(expected_bbox_4326, wgs84_bbox, 0.00001) From 779b508a2c566b45eed650ae6a3c83221adb44fc Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 14:04:49 +0200 Subject: [PATCH 15/17] Add get_run_after_chain_queue method --- src/layman/celery.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/layman/celery.py b/src/layman/celery.py index caf1d75bb..2c5ff9595 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -73,6 +73,15 @@ def pop_step_to_run_after_chain(workspace, publication_type, publication_name, ) return result +def get_run_after_chain_queue(workspace, publication_type, publication_name, ): + rds = settings.LAYMAN_REDIS + key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION + hash = _get_publication_hash(workspace, publication_type, publication_name) + val = rds.hget(key, hash) + queue = json.loads(val) if val is not None else list() + return queue + + def clear_steps_to_run_after_chain(workspace, publication_type, publication_name, ): rds = settings.LAYMAN_REDIS key = LAST_TASK_ID_IN_CHAIN_TO_PUBLICATION From 8322b0aec8e9d65e4e0e3785dc105fc4cb101414 Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 14:05:45 +0200 Subject: [PATCH 16/17] Improve test_wfst_concurrency test --- src/layman/requests_concurrency_test.py | 33 ++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/layman/requests_concurrency_test.py b/src/layman/requests_concurrency_test.py index 2d3cf6909..304668812 100644 --- a/src/layman/requests_concurrency_test.py +++ b/src/layman/requests_concurrency_test.py @@ -1,9 +1,11 @@ -from test import process_client +import time + +from test import process_client, assert_util from test.data import wfs as data_wfs import requests import pytest -from layman import settings +from layman import celery, settings from layman.common import empty_method_returns_true @@ -19,23 +21,48 @@ def test_wfst_concurrency(): 'Content-type': 'text/xml', } - process_client.publish_workspace_layer(workspace, layer,) + process_client.publish_workspace_layer(workspace, layer, ) + + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert not queue r = requests.post(rest_url, data=data_xml, headers=headers) assert r.status_code == 200, r.text + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert len(queue) == 0, queue + process_client.patch_workspace_layer(workspace, layer, title='New title', check_response_fn=empty_method_returns_true) + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert len(queue) == 1, queue + assert queue == ['layman.util::patch_after_wfst', ] r = requests.post(rest_url, data=data_xml, headers=headers) assert r.status_code == 200, r.text + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert len(queue) == 1, queue + assert queue == ['layman.util::patch_after_wfst', ] + r = requests.post(rest_url, data=data_xml, headers=headers) assert r.status_code == 200, r.text + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert len(queue) == 1, queue + assert queue == ['layman.util::patch_after_wfst', ] + + time.sleep(3) + + expected_bbox = (1571000.0, 6268800.0, 1572590.8542062, 6269876.33561699) + assert_util.assert_all_sources_bbox(workspace, layer, expected_bbox) + process_client.delete_workspace_layer(workspace, layer, ) + + queue = celery.get_run_after_chain_queue(workspace, process_client.LAYER_TYPE, layer) + assert not queue, queue From 7c636276f9c855b0729533254f4892bff9fad71a Mon Sep 17 00:00:00 2001 From: index-git <66255344+index-git@users.noreply.github.com> Date: Sat, 8 May 2021 19:14:51 +0200 Subject: [PATCH 17/17] Make layman.util::patch_after_wfst the main and call it from layman.layer.util::patch_after_wfst --- src/layman/layer/util.py | 6 +----- src/layman/util.py | 12 +++++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index 8290ef0a6..b77727587 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -194,11 +194,7 @@ def patch_layer(workspace, layername, task_options, stop_sync_at, start_async_at def patch_after_wfst(workspace, layername, **kwargs): - task_methods = tasks_util.get_source_task_methods(get_layer_type_def(), 'patch_after_wfst') - patch_chain = tasks_util.get_chain_of_methods(workspace, layername, task_methods, kwargs, 'layername') - res = patch_chain() - - celery_util.set_publication_chain_info(workspace, LAYER_TYPE, layername, task_methods, res) + layman_util.patch_after_wfst(workspace, LAYER_TYPE, layername, **kwargs) def delete_layer(workspace, layername, source=None, http_method='delete'): diff --git a/src/layman/util.py b/src/layman/util.py index 19678b059..900456bb6 100644 --- a/src/layman/util.py +++ b/src/layman/util.py @@ -10,7 +10,8 @@ from flask import current_app, request, url_for as flask_url_for, jsonify from unidecode import unidecode -from layman import settings +from layman import settings, celery as celery_util +from layman.common import tasks as tasks_util from layman.http import LaymanError logger = logging.getLogger(__name__) @@ -389,7 +390,8 @@ def delete_publications(user, return jsonify(infos) -def patch_after_wfst(workspace, publication_type, layername, ): - from layman.layer import LAYER_TYPE, util as layer_util - if publication_type == LAYER_TYPE: - layer_util.patch_after_wfst(workspace, layername) +def patch_after_wfst(workspace, publication_type, publication, **kwargs): + task_methods = tasks_util.get_source_task_methods(get_publication_types()[publication_type], 'patch_after_wfst') + patch_chain = tasks_util.get_chain_of_methods(workspace, publication, task_methods, kwargs, 'layername') + res = patch_chain() + celery_util.set_publication_chain_info(workspace, publication_type, publication, task_methods, res)