From f83833dc019d082fa7ccf6e324d83e924e4c0319 Mon Sep 17 00:00:00 2001 From: index-git Date: Tue, 18 May 2021 13:30:44 +0200 Subject: [PATCH] Make sure, publication is deleted even if asynchronous tasks are still running --- src/layman/celery.py | 10 +++++++++- src/layman/geoserver_proxy_test.py | 2 -- src/layman/layer/empty_bbox_test.py | 2 -- src/layman/layer/micka/csw.py | 6 ++++-- src/layman/layer/micka/soap_tasks.py | 13 ++++++++++++- src/layman/util.py | 12 ++++++++++++ 6 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/layman/celery.py b/src/layman/celery.py index 59d7f2695..659ddaddd 100644 --- a/src/layman/celery.py +++ b/src/layman/celery.py @@ -6,7 +6,7 @@ from celery.contrib.abortable import AbortableAsyncResult from layman.publication_relation.util import update_related_publications_after_change -from layman import settings, common +from layman import settings, common, util as layman_util from layman.common import redis as redis_util REDIS_CURRENT_TASK_NAMES = f"{__name__}:CURRENT_TASK_NAMES" @@ -47,6 +47,14 @@ def task_postrun(workspace, publication_type, publication_name, task_id, task_na last_task_id = chain_info['last'] finish_publication_chain(last_task_id) clear_steps_to_run_after_chain(workspace, publication_type, publication_name) + # Sometimes, when delete request run just after other request for the same publication (for example WFS-T), + # the aborted task keep running and finish after end of delete task for the same source. This part make sure, + # that in that case we delete it. + info = layman_util.get_publication_info(workspace, publication_type, publication_name, context={'keys': ['name']}) + if not info: + current_app.logger.warning(f"POST task={task_name}, workspace={workspace}, publication_type={publication_type}," + f"publication_name={publication_name} Publication does not exist, so we delete it") + layman_util.delete_workspace_publication(workspace, publication_type, publication_name) def _get_task_hash(task_name, workspace, publication_name): diff --git a/src/layman/geoserver_proxy_test.py b/src/layman/geoserver_proxy_test.py index 20313955d..a43b02187 100644 --- a/src/layman/geoserver_proxy_test.py +++ b/src/layman/geoserver_proxy_test.py @@ -238,7 +238,6 @@ def wfs_post(workspace, attr_names_list, data_xml): data_xml = data_wfs.get_wfs11_insert_polygon_new_attr(username, layername, attr_names10) wfs_post(username, [(layername, attr_names10)], data_xml) - process_client.wait_for_publication_status(username, process_client.LAYER_TYPE, layername, headers=authn_headers) process_client.delete_workspace_layer(username, layername, headers=authn_headers) process_client.delete_workspace_layer(username, layername2, headers=authn_headers) @@ -293,7 +292,6 @@ def do_test(wfs_query, attribute_names): data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername1, attr_names) do_test(data_xml, attr_names) - process_client.wait_for_publication_status(username, process_client.LAYER_TYPE, layername1, headers=authn_headers1) process_client.delete_workspace_layer(username, layername1, headers=authn_headers1) diff --git a/src/layman/layer/empty_bbox_test.py b/src/layman/layer/empty_bbox_test.py index 895d59d97..92ac61b77 100644 --- a/src/layman/layer/empty_bbox_test.py +++ b/src/layman/layer/empty_bbox_test.py @@ -59,6 +59,4 @@ def test_empty_shapefile(layername, file_paths): assert wms_layer.boundingBox == native_bbox assert wms_layer.boundingBoxWGS84 == wgs_bbox - process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layername) - process_client.delete_workspace_layer(workspace, layername) diff --git a/src/layman/layer/micka/csw.py b/src/layman/layer/micka/csw.py index f5de11896..37547976a 100644 --- a/src/layman/layer/micka/csw.py +++ b/src/layman/layer/micka/csw.py @@ -87,8 +87,10 @@ def patch_layer(workspace, layername, metadata_properties_to_refresh, _actor_nam return muuid -def delete_layer(workspace, layername): - uuid = get_layer_uuid(workspace, layername) +def delete_layer(workspace, layername, *, backup_uuid=None): + uuid = get_layer_uuid(workspace, layername) or backup_uuid + if backup_uuid and uuid: + assert backup_uuid == uuid muuid = get_metadata_uuid(uuid) if muuid is None: return diff --git a/src/layman/layer/micka/soap_tasks.py b/src/layman/layer/micka/soap_tasks.py index ea916bec4..5b1d7a10c 100644 --- a/src/layman/layer/micka/soap_tasks.py +++ b/src/layman/layer/micka/soap_tasks.py @@ -1,8 +1,9 @@ from celery.utils.log import get_task_logger from layman.celery import AbortedException -from layman import celery_app +from layman import celery_app, util as layman_util from . import soap +from .. import LAYER_TYPE logger = get_task_logger(__name__) @@ -19,8 +20,18 @@ def patch_after_feature_change( ): if self.is_aborted(): raise AbortedException + uuid = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['uuid']})['uuid'] soap.patch_layer(workspace, layer, metadata_properties_to_refresh=['extent']) + # Sometimes, when delete request run just after other request for the same publication (for example WFS-T), + # the aborted task keep running and finish after end of delete task for the same source. This part make sure, + # that in that case we delete it. + info = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['name']}) + if not info: + logger.warning(f"layman.layer.micka.soap.patch_after_feature_change: workspace={workspace}, " + f"layer={layer}, uuid={uuid} Publication does not exist, so we delete it") + soap.delete_layer(workspace, layer, backup_uuid=uuid) + if self.is_aborted(): raise AbortedException diff --git a/src/layman/util.py b/src/layman/util.py index 44ae3b91f..59cf6d44b 100644 --- a/src/layman/util.py +++ b/src/layman/util.py @@ -347,6 +347,18 @@ def get_publication_infos_with_metainfo(workspace=None, publ_type=None, context= return infos +def delete_workspace_publication(workspace, publication_type, publication): + from layman.layer import LAYER_TYPE, util as layer_util + from layman.map import MAP_TYPE, util as map_util + + delete_method = { + LAYER_TYPE: layer_util.delete_layer, + MAP_TYPE: map_util.delete_map, + }[publication_type] + + delete_method(workspace, publication) + + def delete_publications(user, publ_type, error_code,