process_client optionally raises if status is not complete

src/layman/error_list.py

@@ -55,4 +55,5 @@
     52: (400, 'GeoServer HTTP or connection error'),
     53: (500, 'Error when publishing on GeoServer. It happens for example for raster files with wrong explicit CRS.'),
     54: (400, 'Wrong header value'),
+    55: (400, 'Publication is not complete'),  # raised by process_client only
 }

src/layman/layer/client_test.py

@@ -92,7 +92,8 @@ def test_post_layers_chunk(browser):
     try:
         process_client.wait_for_publication_status(WORKSPACE,
                                                    process_client.LAYER_TYPE,
-                                                   LAYERNAME)
+                                                   LAYERNAME,
+                                                   raise_if_not_complete=True)
     except Exception as exc:
         browser.save_screenshot('/code/tmp/artifacts/client-post-layers-2.5.png')
         raise exc
@@ -171,7 +172,8 @@ def test_patch_layer_chunk(browser):
     try:
         process_client.wait_for_publication_status(WORKSPACE,
                                                    process_client.LAYER_TYPE,
-                                                   LAYERNAME)
+                                                   LAYERNAME,
+                                                   raise_if_not_complete=True)
     except Exception as exc:
         browser.save_screenshot('/code/tmp/artifacts/client-patch-layers-3.5.png')
         raise exc

src/layman/layer/filesystem/thumbnail_test.py

@@ -31,6 +31,7 @@ def wait_for_thumbnail_error(response):
                                            file_paths=geojson_file,
                                            style_file=style_file,
                                            check_response_fn=wait_for_thumbnail_error,
+                                           raise_if_not_complete=False,
                                            )
 
     layer_info = process_client.get_workspace_layer(workspace, layer)

src/layman/layer/rest_workspace_test.py

@@ -958,7 +958,8 @@ def wait_for_db_finish(response):
         info = response.json()
         return info.get('db', {}).get('status', '') == 'FAILURE'
 
-    process_client.publish_workspace_layer(workspace, layername, file_paths=file_paths, check_response_fn=wait_for_db_finish)
+    process_client.publish_workspace_layer(workspace, layername, file_paths=file_paths,
+                                           check_response_fn=wait_for_db_finish, raise_if_not_complete=False)
 
     layer_info = util.get_layer_info(workspace, layername)
     assert layer_info['db']['status'] == 'FAILURE', f'layer_info={layer_info}'

src/layman/publication_relation/layer_map_relation_test.py

@@ -7,7 +7,7 @@
 
 
 def assert_map_thumbnail(workspace, map, expected_thumbnail_path):
-    process_client.wait_for_publication_status(workspace, process_client.MAP_TYPE, map)
+    process_client.wait_for_publication_status(workspace, process_client.MAP_TYPE, map, raise_if_not_complete=True)
     with app.app_context():
         thumbnail_path = thumbnail.get_map_thumbnail_path(workspace, map)
     diffs = test_util.compare_images(expected_thumbnail_path, thumbnail_path)
@@ -34,12 +34,14 @@ def test_map_refresh_after_layer_change():
     # Test refresh map thumbnail after layer WFS-T query
     data_xml = data_wfs.get_wfs20_insert_points(workspace, layer, )
     process_client.post_wfst(data_xml)
-    process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer)
+    process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer,
+                                               raise_if_not_complete=True)
     assert_map_thumbnail(workspace, map, f'/code/test_tools/data/thumbnail/map_with_internal_layer_basic_after_wfst.png')
 
     # Test refresh map thumbnail after patch layer
     process_client.patch_workspace_layer(workspace, layer, file_paths=['sample/layman.layer/small_layer.geojson'])
-    process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer)
+    process_client.wait_for_publication_status(workspace, process_client.LAYER_TYPE, layer,
+                                               raise_if_not_complete=True)
     assert_map_thumbnail(workspace, map, f'/code/test_tools/data/thumbnail/map_with_internal_layer_basic.png')
 
     process_client.delete_workspace_map(workspace, map)

src/layman/requests_concurrency_test.py

@@ -25,7 +25,8 @@ def test_patch_after_feature_change_concurrency(publication_type):
     assert lock == common_const.PUBLICATION_LOCK_FEATURE_CHANGE
 
     process_client.patch_workspace_publication(publication_type, workspace, publication, title='New title',
-                                               check_response_fn=empty_method_returns_true)
+                                               check_response_fn=empty_method_returns_true,
+                                               raise_if_not_complete=False)
     queue = celery.get_run_after_chain_queue(workspace, publication_type, publication)
     assert len(queue) == 0, queue
     lock = redis.get_publication_lock(workspace, publication_type, publication)
@@ -45,7 +46,8 @@ def test_patch_after_feature_change_concurrency(publication_type):
     lock = redis.get_publication_lock(workspace, publication_type, publication)
     assert lock == common_const.PUBLICATION_LOCK_FEATURE_CHANGE
 
-    process_client.wait_for_publication_status(workspace, publication_type, publication)
+    process_client.wait_for_publication_status(workspace, publication_type, publication,
+                                               raise_if_not_complete=True)
     queue = celery.get_run_after_chain_queue(workspace, publication_type, publication)
     assert not queue, queue
     lock = redis.get_publication_lock(workspace, publication_type, publication)

src/layman/rest_publication_test.py

@@ -119,7 +119,8 @@ def test_get_publication_layman_status(publ_type, error_params):
     workspace = 'test_get_publication_layman_status_workspace'
     publication = 'test_get_publication_layman_status_publication'
 
-    process_client.publish_workspace_publication(publ_type, workspace, publication, check_response_fn=common.empty_method_returns_true,)
+    process_client.publish_workspace_publication(publ_type, workspace, publication, check_response_fn=common.empty_method_returns_true,
+                                                 raise_if_not_complete=False)
 
     info = process_client.get_workspace_publication(publ_type, workspace, publication,)
     assert 'layman_metadata' in info, f'info={info}'
@@ -134,7 +135,8 @@ def test_get_publication_layman_status(publ_type, error_params):
     assert info['layman_metadata']['publication_status'] == 'COMPLETE', f'info={info}'
 
     if error_params:
-        process_client.patch_workspace_publication(publ_type, workspace, publication, **error_params, )
+        process_client.patch_workspace_publication(publ_type, workspace, publication, **error_params,
+                                                   raise_if_not_complete=False)
         info = process_client.get_workspace_publication(publ_type, workspace, publication, )
         assert 'layman_metadata' in info, f'info={info}'
         assert 'publication_status' in info['layman_metadata'], f'info={info}'

src/layman/upgrade/upgrade_v1_21_test.py

@@ -31,6 +31,7 @@
             'compress': True,
             'with_chunks': True,
             'do_not_upload_chunks': True,
+            'raise_if_not_complete': False,
         },
         settings.EnumWfsWmsStatus.NOT_AVAILABLE,
         id='layer_updating',
@@ -47,6 +48,7 @@
             ],
             'compress': True,
             'with_chunks': True,
+            'raise_if_not_complete': False,
         },
         settings.EnumWfsWmsStatus.NOT_AVAILABLE,
         id='layer_not_available',

test_tools/process_client.py

@@ -114,6 +114,7 @@ def wait_for_rest(url, max_attempts, sleeping_time, check_response, headers=None
         if attempts > max_attempts:
             logger.error(f"r.status_code={response.status_code}\nrltest={response.text}")
             raise Exception('Max attempts reached!')
+    return response
 
 
 def raise_layman_error(response, status_codes_to_skip=None):
@@ -132,6 +133,20 @@ def raise_layman_error(response, status_codes_to_skip=None):
     assert 'Deprecation' not in response.headers, f'This is deprecated URL! Use new one. headers={response.headers}'
 
 
+def raise_if_not_complete_status(response):
+    resp_json = response.json()
+    status = resp_json.get('layman_metadata', {}).get('publication_status')
+    if status != 'COMPLETE':
+        failed_source_key = next((k for k, v in resp_json.items() if isinstance(v, dict) and v.get('status') == 'FAILURE'), None)
+        if failed_source_key and resp_json[failed_source_key].get('error').get('code'):
+            failed_source = resp_json[failed_source_key]
+            error_desc = failed_source['error']
+            raise LaymanError(error_desc['code'],
+                              error_desc.get('detail'),
+                              sub_code=error_desc.get('sub_code'))
+        raise LaymanError(55, data=resp_json)
+
+
 def upload_file_chunks(publication_type,
                        workspace,
                        name,
@@ -177,6 +192,7 @@ def patch_workspace_publication(publication_type,
                                 title=None,
                                 style_file=None,
                                 check_response_fn=None,
+                                raise_if_not_complete=True,
                                 compress=False,
                                 compress_settings=None,
                                 with_chunks=False,
@@ -201,6 +217,8 @@ def patch_workspace_publication(publication_type,
 
         assert not (not with_chunks and do_not_upload_chunks)
         assert not (check_response_fn and do_not_upload_chunks)  # because check_response_fn is not called when do_not_upload_chunks
+        assert not (raise_if_not_complete and do_not_upload_chunks)
+        assert not (check_response_fn and raise_if_not_complete)
 
         assert not (time_regex and publication_type == MAP_TYPE)
         assert not (publication_type == LAYER_TYPE and crs and not file_paths)
@@ -281,7 +299,8 @@ def patch_workspace_publication(publication_type,
                            file_paths, )
 
     if not do_not_upload_chunks:
-        wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers)
+        wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn,
+                                    headers=headers, raise_if_not_complete=raise_if_not_complete)
     wfs.clear_cache(workspace)
     wms.clear_cache(workspace)
     if temp_dir:
@@ -337,6 +356,7 @@ def publish_workspace_publication(publication_type,
                                   style_file=None,
                                   description=None,
                                   check_response_fn=None,
+                                  raise_if_not_complete=True,
                                   with_chunks=False,
                                   compress=False,
                                   compress_settings=None,
@@ -360,6 +380,8 @@ def publish_workspace_publication(publication_type,
 
     assert not (not with_chunks and do_not_upload_chunks)
     assert not (check_response_fn and do_not_upload_chunks)  # because check_response_fn is not called when do_not_upload_chunks
+    assert not (raise_if_not_complete and do_not_upload_chunks)
+    assert not (check_response_fn and raise_if_not_complete)
 
     file_paths = [publication_type_def.source_path] if file_paths is None and external_table_uri is None and not map_layers else file_paths
 
@@ -440,7 +462,8 @@ def publish_workspace_publication(publication_type,
                            file_paths, )
 
     if not do_not_upload_chunks:
-        wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers)
+        wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn,
+                                    headers=headers, raise_if_not_complete=raise_if_not_complete)
     if temp_dir:
         shutil.rmtree(temp_dir)
     return response.json()[0]
@@ -661,14 +684,17 @@ def check_publication_status(response):
     return current_status in {'COMPLETE', 'INCOMPLETE'}
 
 
-def wait_for_publication_status(workspace, publication_type, publication, *, check_response_fn=None, headers=None,):
+def wait_for_publication_status(workspace, publication_type, publication, *, check_response_fn=None, headers=None,
+                                raise_if_not_complete=False):
     publication_type_def = PUBLICATION_TYPES_DEF[publication_type]
     with app.app_context():
         url = url_for(publication_type_def.get_workspace_publication_url,
                       workspace=workspace,
                       **{publication_type_def.url_param_name: publication})
     check_response_fn = check_response_fn or check_publication_status
-    wait_for_rest(url, 60, 0.5, check_response=check_response_fn, headers=headers)
+    response = wait_for_rest(url, 60, 0.5, check_response=check_response_fn, headers=headers)
+    if raise_if_not_complete:
+        raise_if_not_complete_status(response)
 
 
 def patch_after_feature_change(workspace, publ_type, name):

test_tools/wfs_client.py

@@ -33,4 +33,5 @@ def post_wfst(workspace, publ_type, name,
     process_client.post_wfst(data_xml, headers=request_headers, url=request_url, workspace=request_workspace)
 
     if wait_for_update:
-        process_client.wait_for_publication_status(workspace, publ_type, name, headers=request_headers)
+        process_client.wait_for_publication_status(workspace, publ_type, name, headers=request_headers,
+                                                   raise_if_not_complete=True)

tests/dynamic_data/publications/celery.py

@@ -15,6 +15,7 @@ def generate(workspace):
                             'tmp/naturalearth/10m/cultural/ne_10m_admin_0_countries.geojson',
                         ],
                         'check_response_fn': empty_method_returns_true,
+                        'raise_if_not_complete': False,
 
                     }),
                     consts.KEY_RESPONSE_ASSERTS: [

tests/dynamic_data/publications/layer_wfst/new_attribute_test.py

@@ -277,7 +277,7 @@ def test_new_attribute(self, layer: Publication, rest_args, params, parametrizat
         process_client.post_wfst(wfst_data, headers=AUTHN_HEADERS, workspace=workspace)
         for layer_name, _ in new_attributes:
             process_client.wait_for_publication_status(workspace, self.publication_type, layer_name,
-                                                       headers=AUTHN_HEADERS)
+                                                       headers=AUTHN_HEADERS, raise_if_not_complete=True)
             assert_publ_util.is_publication_valid_and_complete(layer)
 
         # assert that new attributes are present

tests/dynamic_data/publications/layer_wfst/refresh_test.py

@@ -80,7 +80,8 @@ def test_refresh(self, layer: Publication, rest_args, parametrization: base_test
         for wfs_method, exp_bbox, exp_native_bbox, thumbnail_bbox_postfix in wfst_actions:
             data_xml = wfs_method(layer.workspace, layer.name, )
             process_client.post_wfst(data_xml, workspace=layer.workspace)
-            process_client.wait_for_publication_status(layer.workspace, process_client.LAYER_TYPE, layer.name)
+            process_client.wait_for_publication_status(layer.workspace, process_client.LAYER_TYPE, layer.name,
+                                                       raise_if_not_complete=True)
             assert_publ_util.is_publication_valid_and_complete(layer)
 
             assert_util.assert_all_sources_bbox(layer.workspace, layer.name, exp_bbox,

tests/dynamic_data/publications/updating_layer_test.py

@@ -27,6 +27,7 @@ class TestUpdatingLayer(base_test.TestSingleRestPublication):
                                          params={'compress': True,
                                                  'with_chunks': True,
                                                  'do_not_upload_chunks': True,
+                                                 'raise_if_not_complete': False,
                                                  }
                                          )]
 

tests/dynamic_data/publications/x_forwarded_prefix/map_file_test.py

@@ -36,6 +36,7 @@ class TestPublication(base_test.TestSingleRestPublication):
     def before_class(self):
         self.post_publication(MAP, args={
             'file_paths': [MAP_FILE_PATH],
+            'raise_if_not_complete': False,  # timgen fails, because one URL points to non-existent service
         }, scope='class')
 
     @pytest.mark.parametrize('headers, exp_url_prefix', [

tests/static_data/data.py

@@ -91,14 +91,17 @@ def publish_publications_step(publications_set, step_num):
     for workspace, publ_type, publication in publications_set:
         data_def = data.PUBLICATIONS[(workspace, publ_type, publication)][data.DEFINITION]
         params = data_def[step_num]
-        write_method(publ_type, workspace, publication, **params, check_response_fn=empty_method_returns_true)
+        write_method(publ_type, workspace, publication, **params, check_response_fn=empty_method_returns_true,
+                     raise_if_not_complete=False)
         if len(data_def) == step_num + 1:
             done_publications.add((workspace, publ_type, publication))
     for workspace, publ_type, publication in publications_set:
         params = data.PUBLICATIONS[(workspace, publ_type, publication)][data.DEFINITION][step_num]
         headers = params.get('headers')
         try:
-            process_client.wait_for_publication_status(workspace, publ_type, publication, headers=headers, check_response_fn=check_publication_status)
+            process_client.wait_for_publication_status(workspace, publ_type, publication, headers=headers,
+                                                       check_response_fn=check_publication_status,
+                                                       raise_if_not_complete=True)
         except AssertionError as ex:
             print(f"AssertionError in publication {workspace, publ_type, publication}, step_num={step_num}.")
             raise ex