From f3671844461ba3e84ef820cfad4b87d6289fff9f Mon Sep 17 00:00:00 2001 From: Jiri Kozel Date: Wed, 22 Nov 2023 13:50:33 +0100 Subject: [PATCH] process_client optionally raises if status is not complete --- src/layman/error_list.py | 1 + src/layman/layer/client_test.py | 6 ++-- src/layman/layer/filesystem/thumbnail_test.py | 1 + src/layman/layer/rest_workspace_test.py | 3 +- .../layer_map_relation_test.py | 8 +++-- src/layman/requests_concurrency_test.py | 6 ++-- src/layman/rest_publication_test.py | 6 ++-- src/layman/upgrade/upgrade_v1_21_test.py | 2 ++ test_tools/process_client.py | 34 ++++++++++++++++--- test_tools/wfs_client.py | 3 +- tests/dynamic_data/publications/celery.py | 1 + .../layer_wfst/new_attribute_test.py | 2 +- .../publications/layer_wfst/refresh_test.py | 3 +- .../publications/updating_layer_test.py | 1 + .../x_forwarded_prefix/map_file_test.py | 1 + tests/static_data/data.py | 7 ++-- 16 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/layman/error_list.py b/src/layman/error_list.py index b0eccf9bb..53d08faea 100644 --- a/src/layman/error_list.py +++ b/src/layman/error_list.py @@ -55,4 +55,5 @@ 52: (400, 'GeoServer HTTP or connection error'), 53: (500, 'Error when publishing on GeoServer. It happens for example for raster files with wrong explicit CRS.'), 54: (400, 'Wrong header value'), + 55: (400, 'Publication is not complete'), # raised by process_client only } diff --git a/src/layman/layer/client_test.py b/src/layman/layer/client_test.py index 923061e19..bb589f917 100644 --- a/src/layman/layer/client_test.py +++ b/src/layman/layer/client_test.py @@ -92,7 +92,8 @@ def test_post_layers_chunk(browser): try: process_client.wait_for_publication_status(WORKSPACE, process_client.LAYER_TYPE, - LAYERNAME) + LAYERNAME, + raise_if_not_complete=True) except Exception as exc: browser.save_screenshot('/code/tmp/artifacts/client-post-layers-2.5.png') raise exc @@ -171,7 +172,8 @@ def test_patch_layer_chunk(browser): try: process_client.wait_for_publication_status(WORKSPACE, process_client.LAYER_TYPE, - LAYERNAME) + LAYERNAME, + raise_if_not_complete=True) except Exception as exc: browser.save_screenshot('/code/tmp/artifacts/client-patch-layers-3.5.png') raise exc diff --git a/src/layman/layer/filesystem/thumbnail_test.py b/src/layman/layer/filesystem/thumbnail_test.py index b82d7bbb4..2c6606a71 100644 --- a/src/layman/layer/filesystem/thumbnail_test.py +++ b/src/layman/layer/filesystem/thumbnail_test.py @@ -31,6 +31,7 @@ def wait_for_thumbnail_error(response): file_paths=geojson_file, style_file=style_file, check_response_fn=wait_for_thumbnail_error, + raise_if_not_complete=False, ) layer_info = process_client.get_workspace_layer(workspace, layer) diff --git a/src/layman/layer/rest_workspace_test.py b/src/layman/layer/rest_workspace_test.py index 9faac9429..af695ef13 100644 --- a/src/layman/layer/rest_workspace_test.py +++ b/src/layman/layer/rest_workspace_test.py @@ -958,7 +958,8 @@ def wait_for_db_finish(response): info = response.json() return info.get('db', {}).get('status', '') == 'FAILURE' - process_client.publish_workspace_layer(workspace, layername, file_paths=file_paths, check_response_fn=wait_for_db_finish) + process_client.publish_workspace_layer(workspace, layername, file_paths=file_paths, + check_response_fn=wait_for_db_finish, raise_if_not_complete=False) layer_info = util.get_layer_info(workspace, layername) assert layer_info['db']['status'] == 'FAILURE', f'layer_info={layer_info}' diff --git a/src/layman/publication_relation/layer_map_relation_test.py b/src/layman/publication_relation/layer_map_relation_test.py index 5db8eec4d..2d808b9a7 100644 --- a/src/layman/publication_relation/layer_map_relation_test.py +++ b/src/layman/publication_relation/layer_map_relation_test.py @@ -7,7 +7,7 @@ def assert_map_thumbnail(workspace, map, expected_thumbnail_path): - process_client.wait_for_publication_status(workspace, process_client.MAP_TYPE, map) + process_client.wait_for_publication_status(workspace, process_client.MAP_TYPE, map, raise_if_not_complete=True) with app.app_context(): thumbnail_path = thumbnail.get_map_thumbnail_path(workspace, map) diffs = test_util.compare_images(expected_thumbnail_path, thumbnail_path) @@ -34,12 +34,14 @@ def test_map_refresh_after_layer_change(): # Test refresh map thumbnail after layer WFS-T query data_xml = data_wfs.get_wfs20_insert_points(workspace, layer, ) process_client.post_wfst(data_xml) - process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer) + process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer, + raise_if_not_complete=True) assert_map_thumbnail(workspace, map, f'/code/test_tools/data/thumbnail/map_with_internal_layer_basic_after_wfst.png') # Test refresh map thumbnail after patch layer process_client.patch_workspace_layer(workspace, layer, file_paths=['sample/layman.layer/small_layer.geojson']) - process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer) + process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer, + raise_if_not_complete=True) assert_map_thumbnail(workspace, map, f'/code/test_tools/data/thumbnail/map_with_internal_layer_basic.png') process_client.delete_workspace_map(workspace, map) diff --git a/src/layman/requests_concurrency_test.py b/src/layman/requests_concurrency_test.py index ccce69914..4914e3dbe 100644 --- a/src/layman/requests_concurrency_test.py +++ b/src/layman/requests_concurrency_test.py @@ -25,7 +25,8 @@ def test_patch_after_feature_change_concurrency(publication_type): assert lock == common_const.PUBLICATION_LOCK_FEATURE_CHANGE process_client.patch_workspace_publication(publication_type, workspace, publication, title='New title', - check_response_fn=empty_method_returns_true) + check_response_fn=empty_method_returns_true, + raise_if_not_complete=False) queue = celery.get_run_after_chain_queue(workspace, publication_type, publication) assert len(queue) == 0, queue lock = redis.get_publication_lock(workspace, publication_type, publication) @@ -45,7 +46,8 @@ def test_patch_after_feature_change_concurrency(publication_type): lock = redis.get_publication_lock(workspace, publication_type, publication) assert lock == common_const.PUBLICATION_LOCK_FEATURE_CHANGE - process_client.wait_for_publication_status(workspace, publication_type, publication) + process_client.wait_for_publication_status(workspace, publication_type, publication, + raise_if_not_complete=True) queue = celery.get_run_after_chain_queue(workspace, publication_type, publication) assert not queue, queue lock = redis.get_publication_lock(workspace, publication_type, publication) diff --git a/src/layman/rest_publication_test.py b/src/layman/rest_publication_test.py index 1f51258c4..e007b1645 100644 --- a/src/layman/rest_publication_test.py +++ b/src/layman/rest_publication_test.py @@ -119,7 +119,8 @@ def test_get_publication_layman_status(publ_type, error_params): workspace = 'test_get_publication_layman_status_workspace' publication = 'test_get_publication_layman_status_publication' - process_client.publish_workspace_publication(publ_type, workspace, publication, check_response_fn=common.empty_method_returns_true,) + process_client.publish_workspace_publication(publ_type, workspace, publication, check_response_fn=common.empty_method_returns_true, + raise_if_not_complete=False) info = process_client.get_workspace_publication(publ_type, workspace, publication,) assert 'layman_metadata' in info, f'info={info}' @@ -134,7 +135,8 @@ def test_get_publication_layman_status(publ_type, error_params): assert info['layman_metadata']['publication_status'] == 'COMPLETE', f'info={info}' if error_params: - process_client.patch_workspace_publication(publ_type, workspace, publication, **error_params, ) + process_client.patch_workspace_publication(publ_type, workspace, publication, **error_params, + raise_if_not_complete=False) info = process_client.get_workspace_publication(publ_type, workspace, publication, ) assert 'layman_metadata' in info, f'info={info}' assert 'publication_status' in info['layman_metadata'], f'info={info}' diff --git a/src/layman/upgrade/upgrade_v1_21_test.py b/src/layman/upgrade/upgrade_v1_21_test.py index 9a965f68d..e1fd8fd63 100644 --- a/src/layman/upgrade/upgrade_v1_21_test.py +++ b/src/layman/upgrade/upgrade_v1_21_test.py @@ -31,6 +31,7 @@ 'compress': True, 'with_chunks': True, 'do_not_upload_chunks': True, + 'raise_if_not_complete': False, }, settings.EnumWfsWmsStatus.NOT_AVAILABLE, id='layer_updating', @@ -47,6 +48,7 @@ ], 'compress': True, 'with_chunks': True, + 'raise_if_not_complete': False, }, settings.EnumWfsWmsStatus.NOT_AVAILABLE, id='layer_not_available', diff --git a/test_tools/process_client.py b/test_tools/process_client.py index 91fbf643f..e5d816c39 100644 --- a/test_tools/process_client.py +++ b/test_tools/process_client.py @@ -114,6 +114,7 @@ def wait_for_rest(url, max_attempts, sleeping_time, check_response, headers=None if attempts > max_attempts: logger.error(f"r.status_code={response.status_code}\nrltest={response.text}") raise Exception('Max attempts reached!') + return response def raise_layman_error(response, status_codes_to_skip=None): @@ -132,6 +133,20 @@ def raise_layman_error(response, status_codes_to_skip=None): assert 'Deprecation' not in response.headers, f'This is deprecated URL! Use new one. headers={response.headers}' +def raise_if_not_complete_status(response): + resp_json = response.json() + status = resp_json.get('layman_metadata', {}).get('publication_status') + if status != 'COMPLETE': + failed_source_key = next((k for k, v in resp_json.items() if isinstance(v, dict) and v.get('status') == 'FAILURE'), None) + if failed_source_key and resp_json[failed_source_key].get('error').get('code'): + failed_source = resp_json[failed_source_key] + error_desc = failed_source['error'] + raise LaymanError(error_desc['code'], + error_desc.get('detail'), + sub_code=error_desc.get('sub_code')) + raise LaymanError(55, data=resp_json) + + def upload_file_chunks(publication_type, workspace, name, @@ -177,6 +192,7 @@ def patch_workspace_publication(publication_type, title=None, style_file=None, check_response_fn=None, + raise_if_not_complete=True, compress=False, compress_settings=None, with_chunks=False, @@ -201,6 +217,8 @@ def patch_workspace_publication(publication_type, assert not (not with_chunks and do_not_upload_chunks) assert not (check_response_fn and do_not_upload_chunks) # because check_response_fn is not called when do_not_upload_chunks + assert not (raise_if_not_complete and do_not_upload_chunks) + assert not (check_response_fn and raise_if_not_complete) assert not (time_regex and publication_type == MAP_TYPE) assert not (publication_type == LAYER_TYPE and crs and not file_paths) @@ -281,7 +299,8 @@ def patch_workspace_publication(publication_type, file_paths, ) if not do_not_upload_chunks: - wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers) + wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, + headers=headers, raise_if_not_complete=raise_if_not_complete) wfs.clear_cache(workspace) wms.clear_cache(workspace) if temp_dir: @@ -337,6 +356,7 @@ def publish_workspace_publication(publication_type, style_file=None, description=None, check_response_fn=None, + raise_if_not_complete=True, with_chunks=False, compress=False, compress_settings=None, @@ -360,6 +380,8 @@ def publish_workspace_publication(publication_type, assert not (not with_chunks and do_not_upload_chunks) assert not (check_response_fn and do_not_upload_chunks) # because check_response_fn is not called when do_not_upload_chunks + assert not (raise_if_not_complete and do_not_upload_chunks) + assert not (check_response_fn and raise_if_not_complete) file_paths = [publication_type_def.source_path] if file_paths is None and external_table_uri is None and not map_layers else file_paths @@ -440,7 +462,8 @@ def publish_workspace_publication(publication_type, file_paths, ) if not do_not_upload_chunks: - wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers) + wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, + headers=headers, raise_if_not_complete=raise_if_not_complete) if temp_dir: shutil.rmtree(temp_dir) return response.json()[0] @@ -661,14 +684,17 @@ def check_publication_status(response): return current_status in {'COMPLETE', 'INCOMPLETE'} -def wait_for_publication_status(workspace, publication_type, publication, *, check_response_fn=None, headers=None,): +def wait_for_publication_status(workspace, publication_type, publication, *, check_response_fn=None, headers=None, + raise_if_not_complete=False): publication_type_def = PUBLICATION_TYPES_DEF[publication_type] with app.app_context(): url = url_for(publication_type_def.get_workspace_publication_url, workspace=workspace, **{publication_type_def.url_param_name: publication}) check_response_fn = check_response_fn or check_publication_status - wait_for_rest(url, 60, 0.5, check_response=check_response_fn, headers=headers) + response = wait_for_rest(url, 60, 0.5, check_response=check_response_fn, headers=headers) + if raise_if_not_complete: + raise_if_not_complete_status(response) def patch_after_feature_change(workspace, publ_type, name): diff --git a/test_tools/wfs_client.py b/test_tools/wfs_client.py index d2f0af0c1..35300a2cf 100644 --- a/test_tools/wfs_client.py +++ b/test_tools/wfs_client.py @@ -33,4 +33,5 @@ def post_wfst(workspace, publ_type, name, process_client.post_wfst(data_xml, headers=request_headers, url=request_url, workspace=request_workspace) if wait_for_update: - process_client.wait_for_publication_status(workspace, publ_type, name, headers=request_headers) + process_client.wait_for_publication_status(workspace, publ_type, name, headers=request_headers, + raise_if_not_complete=True) diff --git a/tests/dynamic_data/publications/celery.py b/tests/dynamic_data/publications/celery.py index 601e1a9c8..a64aa36c1 100644 --- a/tests/dynamic_data/publications/celery.py +++ b/tests/dynamic_data/publications/celery.py @@ -15,6 +15,7 @@ def generate(workspace): 'tmp/naturalearth/10m/cultural/ne_10m_admin_0_countries.geojson', ], 'check_response_fn': empty_method_returns_true, + 'raise_if_not_complete': False, }), consts.KEY_RESPONSE_ASSERTS: [ diff --git a/tests/dynamic_data/publications/layer_wfst/new_attribute_test.py b/tests/dynamic_data/publications/layer_wfst/new_attribute_test.py index d8172cf22..52ba9ec96 100644 --- a/tests/dynamic_data/publications/layer_wfst/new_attribute_test.py +++ b/tests/dynamic_data/publications/layer_wfst/new_attribute_test.py @@ -277,7 +277,7 @@ def test_new_attribute(self, layer: Publication, rest_args, params, parametrizat process_client.post_wfst(wfst_data, headers=AUTHN_HEADERS, workspace=workspace) for layer_name, _ in new_attributes: process_client.wait_for_publication_status(workspace, self.publication_type, layer_name, - headers=AUTHN_HEADERS) + headers=AUTHN_HEADERS, raise_if_not_complete=True) assert_publ_util.is_publication_valid_and_complete(layer) # assert that new attributes are present diff --git a/tests/dynamic_data/publications/layer_wfst/refresh_test.py b/tests/dynamic_data/publications/layer_wfst/refresh_test.py index ba6694038..0ae0c4a4c 100644 --- a/tests/dynamic_data/publications/layer_wfst/refresh_test.py +++ b/tests/dynamic_data/publications/layer_wfst/refresh_test.py @@ -80,7 +80,8 @@ def test_refresh(self, layer: Publication, rest_args, parametrization: base_test for wfs_method, exp_bbox, exp_native_bbox, thumbnail_bbox_postfix in wfst_actions: data_xml = wfs_method(layer.workspace, layer.name, ) process_client.post_wfst(data_xml, workspace=layer.workspace) - process_client.wait_for_publication_status(layer.workspace, process_client.LAYER_TYPE, layer.name) + process_client.wait_for_publication_status(layer.workspace, process_client.LAYER_TYPE, layer.name, + raise_if_not_complete=True) assert_publ_util.is_publication_valid_and_complete(layer) assert_util.assert_all_sources_bbox(layer.workspace, layer.name, exp_bbox, diff --git a/tests/dynamic_data/publications/updating_layer_test.py b/tests/dynamic_data/publications/updating_layer_test.py index 547bafbc6..d36760d3d 100644 --- a/tests/dynamic_data/publications/updating_layer_test.py +++ b/tests/dynamic_data/publications/updating_layer_test.py @@ -27,6 +27,7 @@ class TestUpdatingLayer(base_test.TestSingleRestPublication): params={'compress': True, 'with_chunks': True, 'do_not_upload_chunks': True, + 'raise_if_not_complete': False, } )] diff --git a/tests/dynamic_data/publications/x_forwarded_prefix/map_file_test.py b/tests/dynamic_data/publications/x_forwarded_prefix/map_file_test.py index b59dae55e..f4b5f6ace 100644 --- a/tests/dynamic_data/publications/x_forwarded_prefix/map_file_test.py +++ b/tests/dynamic_data/publications/x_forwarded_prefix/map_file_test.py @@ -36,6 +36,7 @@ class TestPublication(base_test.TestSingleRestPublication): def before_class(self): self.post_publication(MAP, args={ 'file_paths': [MAP_FILE_PATH], + 'raise_if_not_complete': False, # timgen fails, because one URL points to non-existent service }, scope='class') @pytest.mark.parametrize('headers, exp_url_prefix', [ diff --git a/tests/static_data/data.py b/tests/static_data/data.py index 1608288a5..6ae322e50 100644 --- a/tests/static_data/data.py +++ b/tests/static_data/data.py @@ -91,14 +91,17 @@ def publish_publications_step(publications_set, step_num): for workspace, publ_type, publication in publications_set: data_def = data.PUBLICATIONS[(workspace, publ_type, publication)][data.DEFINITION] params = data_def[step_num] - write_method(publ_type, workspace, publication, **params, check_response_fn=empty_method_returns_true) + write_method(publ_type, workspace, publication, **params, check_response_fn=empty_method_returns_true, + raise_if_not_complete=False) if len(data_def) == step_num + 1: done_publications.add((workspace, publ_type, publication)) for workspace, publ_type, publication in publications_set: params = data.PUBLICATIONS[(workspace, publ_type, publication)][data.DEFINITION][step_num] headers = params.get('headers') try: - process_client.wait_for_publication_status(workspace, publ_type, publication, headers=headers, check_response_fn=check_publication_status) + process_client.wait_for_publication_status(workspace, publ_type, publication, headers=headers, + check_response_fn=check_publication_status, + raise_if_not_complete=True) except AssertionError as ex: print(f"AssertionError in publication {workspace, publ_type, publication}, step_num={step_num}.") raise ex