diff --git a/CHANGELOG.md b/CHANGELOG.md index 499507c89..699399f7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - flask 2.2.2 -> 2.3.2 - redis 4.5.1 -> 4.5.4 - owslib 0.27.2 -> 0.28.1 +- [#520](https://github.com/LayerManager/layman/issues/520) WFS in version `2.0.0` and WMS in version `1.3.0` includes also MetadataURL for each layer with url of Micka record. - [#833](https://github.com/LayerManager/layman/issues/833) Make Timgen WMS requests more robust (handle WML errors, delayed retry, add timestamp to each request). - [#815](https://github.com/LayerManager/layman/issues/815) Propagate [`LAYMAN_PROXY_SERVER_NAME`](doc/env-settings.md#LAYMAN_PROXY_SERVER_NAME) value to GeoServer environment variable [GEOSERVER_CSRF_WHITELIST](https://docs.geoserver.org/latest/en/user/security/webadmin/csrf.html). - [#847](https://github.com/LayerManager/layman/issues/847) Fix publishing external table layers with `@` in the username or the password. diff --git a/src/geoserver/util.py b/src/geoserver/util.py index 44cc21fbe..5bcc02760 100644 --- a/src/geoserver/util.py +++ b/src/geoserver/util.py @@ -273,7 +273,7 @@ def patch_feature_type(geoserver_workspace, feature_type_name, store_name=None, response.raise_for_status() -def post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, auth, *, lat_lon_bbox, table_name, store_name=None): +def post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, auth, *, lat_lon_bbox, table_name, metadata_url, store_name=None, ): store_name = store_name or DEFAULT_DB_STORE_NAME keywords = [ "features", @@ -298,6 +298,15 @@ def post_feature_type(geoserver_workspace, layername, description, title, bbox, }, 'nativeBoundingBox': bbox_to_dict(bbox, crs), 'latLonBoundingBox': bbox_to_dict(lat_lon_bbox, 'CRS:84'), + 'metadataLinks': { + "metadataLink": [ + { + "type": "application/xml", + "metadataType": "ISO19115:2003", + "content": metadata_url, + } + ] + } } response = requests.post(urljoin(GS_REST_WORKSPACES, f'{geoserver_workspace}/datastores/{store_name}/featuretypes/'), @@ -615,7 +624,8 @@ def delete_coverage_store(geoserver_workspace, auth, name): response.raise_for_status() -def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, description, bbox, crs, *, lat_lon_bbox, enable_time_dimension=False): +def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, description, bbox, crs, *, lat_lon_bbox, metadata_url, + enable_time_dimension=False): keywords = [ "features", layer, @@ -641,6 +651,15 @@ def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, de "name": f"{geoserver_workspace}:{coverage_store}" }, "title": title, + 'metadataLinks': { + "metadataLink": [ + { + "type": "application/xml", + "metadataType": "ISO19115:2003", + "content": metadata_url, + } + ] + } } } if enable_time_dimension: @@ -755,7 +774,7 @@ def patch_wms_layer(geoserver_workspace, layer, *, auth, bbox=None, title=None, response.raise_for_status() -def post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, auth, *, lat_lon_bbox): +def post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, auth, *, lat_lon_bbox, metadata_url): keywords = [ "features", layer, @@ -780,6 +799,15 @@ def post_wms_layer(geoserver_workspace, layer, store_name, title, description, b }, 'nativeBoundingBox': bbox_to_dict(bbox, crs), 'latLonBoundingBox': bbox_to_dict(lat_lon_bbox, 'CRS:84'), + 'metadataLinks': { + "metadataLink": [ + { + "type": "application/xml", + "metadataType": "ISO19115:2003", + "content": metadata_url, + } + ] + } } response = requests.post(urljoin(GS_REST_WORKSPACES, geoserver_workspace + '/wmslayers/'), diff --git a/src/layman/common/micka/util.py b/src/layman/common/micka/util.py index 41fc23ddd..fb7c18d48 100644 --- a/src/layman/common/micka/util.py +++ b/src/layman/common/micka/util.py @@ -1,3 +1,4 @@ +from enum import Enum import os import time from io import BytesIO @@ -19,6 +20,11 @@ logger = logging.getLogger(__name__) +class RecordUrlType(Enum): + BASIC = 'basic' + XML = 'xml' + + for k, v in NAMESPACES.items(): ET.register_namespace(k, v) @@ -27,10 +33,10 @@ def get_metadata_uuid(uuid): return f"m-{uuid}" if uuid is not None else None -def get_metadata_url(uuid): +def get_metadata_url(uuid, *, url_type: RecordUrlType): muuid = get_metadata_uuid(uuid) server_url = settings.CSW_PROXY_URL[:-3] - result = f'{server_url}record/basic/{muuid}' + result = f'{server_url}record/{url_type.value}/{muuid}' return result diff --git a/src/layman/layer/geoserver/__init__.py b/src/layman/layer/geoserver/__init__.py index 9d8ce0cde..225be114a 100644 --- a/src/layman/layer/geoserver/__init__.py +++ b/src/layman/layer/geoserver/__init__.py @@ -123,14 +123,14 @@ def get_layer_native_bbox(workspace, layer): return gs_util.bbox_to_dict(bbox, crs) -def publish_layer_from_db(workspace, layername, description, title, *, crs, table_name, geoserver_workspace=None, store_name=None): +def publish_layer_from_db(workspace, layername, description, title, *, crs, table_name, metadata_url, geoserver_workspace=None, store_name=None): geoserver_workspace = geoserver_workspace or workspace bbox = get_layer_bbox(workspace, layername) lat_lon_bbox = bbox_util.transform(bbox, crs, crs_def.EPSG_4326) - gs_util.post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox, table_name=table_name, store_name=store_name) + gs_util.post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox, table_name=table_name, metadata_url=metadata_url, store_name=store_name) -def publish_layer_from_qgis(workspace, layer, description, title, *, geoserver_workspace=None): +def publish_layer_from_qgis(workspace, layer, description, title, *, metadata_url, geoserver_workspace=None): geoserver_workspace = geoserver_workspace or workspace store_name = wms.get_qgis_store_name(layer) info = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['wms', 'native_crs', ]}) @@ -142,7 +142,8 @@ def publish_layer_from_qgis(workspace, layer, description, title, *, geoserver_w layer_capabilities_url) bbox = get_layer_bbox(workspace, layer) lat_lon_bbox = bbox_util.transform(bbox, crs, crs_def.EPSG_4326) - gs_util.post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox) + gs_util.post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, settings.LAYMAN_GS_AUTH, + lat_lon_bbox=lat_lon_bbox, metadata_url=metadata_url) def get_usernames(): diff --git a/src/layman/layer/geoserver/tasks.py b/src/layman/layer/geoserver/tasks.py index 7743cfd3b..c2b376f1b 100644 --- a/src/layman/layer/geoserver/tasks.py +++ b/src/layman/layer/geoserver/tasks.py @@ -7,6 +7,7 @@ from layman.celery import AbortedException from layman import celery_app, settings, util as layman_util from layman.common import empty_method_returns_true, bbox as bbox_util +from layman.common.micka import util as micka_util from layman.http import LaymanError from . import wms, wfs, sld from .. import geoserver, LAYER_TYPE @@ -38,7 +39,7 @@ def refresh_wms( original_data_source=settings.EnumOriginalDataSource.FILE.value, ): info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': [ - 'file', 'geodata_type', 'native_bounding_box', 'native_crs', 'table_uri' + 'file', 'geodata_type', 'native_bounding_box', 'native_crs', 'table_uri', 'uuid' ]}) geodata_type = info['geodata_type'] crs = info['native_crs'] @@ -47,6 +48,7 @@ def refresh_wms( assert title is not None geoserver_workspace = wms.get_geoserver_workspace(workspace) geoserver.ensure_workspace(workspace) + metadata_url = micka_util.get_metadata_url(info['uuid'], url_type=micka_util.RecordUrlType.XML) if self.is_aborted(): raise AbortedException @@ -67,6 +69,7 @@ def refresh_wms( title, crs=crs, table_name=table_name, + metadata_url=metadata_url, geoserver_workspace=geoserver_workspace, store_name=store_name, ) @@ -75,6 +78,7 @@ def refresh_wms( layername, description, title, + metadata_url=metadata_url, geoserver_workspace=geoserver_workspace, ) elif geodata_type == settings.GEODATA_TYPE_RASTER: @@ -102,7 +106,7 @@ def refresh_wms( enable_time_dimension = True gs_util.create_coverage_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, coverage_store_name, source_file_or_dir, coverage_type=coverage_type) gs_util.publish_coverage(geoserver_workspace, settings.LAYMAN_GS_AUTH, coverage_store_name, layername, title, - description, bbox, crs, lat_lon_bbox=lat_lon_bbox, enable_time_dimension=enable_time_dimension) + description, bbox, crs, lat_lon_bbox=lat_lon_bbox, metadata_url=metadata_url, enable_time_dimension=enable_time_dimension) else: raise NotImplementedError(f"Unknown geodata type: {geodata_type}") @@ -139,7 +143,7 @@ def refresh_wfs( access_rights=None, original_data_source=settings.EnumOriginalDataSource.FILE.value, ): - info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['geodata_type', 'native_crs', 'table_uri']}) + info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['geodata_type', 'native_crs', 'table_uri', 'uuid']}) geodata_type = info['geodata_type'] if geodata_type == settings.GEODATA_TYPE_RASTER: return @@ -161,7 +165,8 @@ def refresh_wfs( layer=layername, table_uri=table_uri, ) - geoserver.publish_layer_from_db(workspace, layername, description, title, crs=crs, table_name=table_name, store_name=store_name) + metadata_url = micka_util.get_metadata_url(info['uuid'], url_type=micka_util.RecordUrlType.XML) + geoserver.publish_layer_from_db(workspace, layername, description, title, crs=crs, table_name=table_name, metadata_url=metadata_url, store_name=store_name) geoserver.set_security_rules(workspace, layername, access_rights, settings.LAYMAN_GS_AUTH, workspace) wfs.clear_cache(workspace) diff --git a/src/layman/layer/micka/csw.py b/src/layman/layer/micka/csw.py index c544cee85..1551ef37c 100644 --- a/src/layman/layer/micka/csw.py +++ b/src/layman/layer/micka/csw.py @@ -49,7 +49,7 @@ def get_layer_info(workspace, layername): 'metadata': { 'identifier': muuid, 'csw_url': settings.CSW_PROXY_URL, - 'record_url': common_util.get_metadata_url(uuid), + 'record_url': common_util.get_metadata_url(uuid, url_type=common_util.RecordUrlType.BASIC), 'comparison_url': url_for('rest_workspace_layer_metadata_comparison.get', workspace=workspace, layername=layername), } } diff --git a/src/layman/map/micka/csw.py b/src/layman/map/micka/csw.py index f1f09c784..b1f4e3951 100644 --- a/src/layman/map/micka/csw.py +++ b/src/layman/map/micka/csw.py @@ -45,7 +45,7 @@ def get_map_info(workspace, mapname): 'metadata': { 'identifier': muuid, 'csw_url': settings.CSW_PROXY_URL, - 'record_url': common_util.get_metadata_url(uuid), + 'record_url': common_util.get_metadata_url(uuid, url_type=common_util.RecordUrlType.BASIC), 'comparison_url': url_for('rest_workspace_map_metadata_comparison.get', workspace=workspace, mapname=mapname), } } diff --git a/src/layman/upgrade/upgrade_v1_17_test.py b/src/layman/upgrade/upgrade_v1_17_test.py index 17ecb3886..69949baa5 100644 --- a/src/layman/upgrade/upgrade_v1_17_test.py +++ b/src/layman/upgrade/upgrade_v1_17_test.py @@ -6,6 +6,7 @@ from layman import app, settings, util as layman_util from layman.common.prime_db_schema import publications as prime_db_schema_publications from layman.common.filesystem import uuid as uuid_common +from layman.common.micka import util as micka_util from layman.layer import LAYER_TYPE, STYLE_TYPES_DEF, db, geoserver, qgis from layman.layer.db import table from layman.layer.geoserver import wms, wfs @@ -150,6 +151,8 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ): geoserver.ensure_workspace(workspace) geoserver.ensure_workspace(wms_workspace) + metadata_url = micka_util.get_metadata_url(uuid_str, url_type=micka_util.RecordUrlType.XML) + # import into GS WFS workspace geoserver.publish_layer_from_db(workspace, layer, @@ -157,6 +160,7 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ): title=layer, crs=crs, table_name=table_name, + metadata_url=metadata_url, geoserver_workspace=workspace, ) @@ -168,6 +172,7 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ): title=layer, crs=crs, table_name=table_name, + metadata_url=metadata_url, geoserver_workspace=wms_workspace, ) elif style_type == 'qml': @@ -195,6 +200,7 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ): layer, description=layer, title=layer, + metadata_url=metadata_url, geoserver_workspace=wms_workspace, ) for gs_workspace in [workspace, wms.get_geoserver_workspace(workspace)]: diff --git a/tests/asserts/final/publication/geoserver.py b/tests/asserts/final/publication/geoserver.py index 100427fe4..2a363f74a 100644 --- a/tests/asserts/final/publication/geoserver.py +++ b/tests/asserts/final/publication/geoserver.py @@ -129,7 +129,7 @@ def is_complete_in_internal_workspace_wms(workspace, publ_type, name): assert publ_type == process_client.LAYER_TYPE wms_inst = wms.get_wms_proxy(workspace) - geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name) + geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name, validate_metadata_url=False) def assert_workspace_stores(workspace, *, exp_stores=None, exp_existing_stores=None, exp_deleted_stores=None): diff --git a/tests/asserts/final/publication/geoserver_proxy.py b/tests/asserts/final/publication/geoserver_proxy.py index 6b81de8cd..c6fe11266 100644 --- a/tests/asserts/final/publication/geoserver_proxy.py +++ b/tests/asserts/final/publication/geoserver_proxy.py @@ -1,5 +1,3 @@ -import requests - from layman import app, settings, util as layman_util from layman.layer.geoserver import util as gs_util from test_tools import util as test_util, process_client @@ -12,7 +10,8 @@ def is_complete_in_workspace_wms(workspace, publ_type, name, *, version, headers with app.app_context(): wms_url = test_util.url_for('geoserver_proxy_bp.proxy', subpath=workspace + settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX + '/ows') wms_inst = gs_util.wms_proxy(wms_url, version=version, headers=headers) - geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name) + validate_metadata_url = version != '1.1.1' + geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name, validate_metadata_url=validate_metadata_url) def is_complete_in_workspace_wms_1_3_0(workspace, publ_type, name, headers): @@ -20,7 +19,7 @@ def is_complete_in_workspace_wms_1_3_0(workspace, publ_type, name, headers): is_complete_in_workspace_wms(workspace, publ_type, name, version='1.3.0', headers=headers) -def workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, name): +def workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, name, headers): with app.app_context(): internal_wfs_url = test_util.url_for('geoserver_proxy_bp.proxy', subpath=workspace + '/wfs') @@ -28,9 +27,11 @@ def workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, n file_info = layman_util.get_publication_info(workspace, publ_type, name, {'keys': ['geodata_type']}) geodata_type = file_info['geodata_type'] if geodata_type == settings.GEODATA_TYPE_VECTOR: - r_wfs = requests.get(internal_wfs_url, params={ - 'service': 'WFS', - 'request': 'GetCapabilities', - 'version': '2.0.0', - }, timeout=settings.DEFAULT_CONNECTION_TIMEOUT) - assert r_wfs.status_code == 200 + wfs_inst = gs_util.wfs_proxy(wfs_url=internal_wfs_url, version='2.0.0', headers=headers) + + assert wfs_inst.contents + wfs_name = f'{workspace}:{name}' + assert wfs_name in wfs_inst.contents + wfs_layer = wfs_inst.contents[wfs_name] + assert len(wfs_layer.metadataUrls) == 1 + assert wfs_layer.metadataUrls[0]['url'].startswith('http://localhost:3080/record/xml/m-') diff --git a/tests/asserts/final/publication/geoserver_util.py b/tests/asserts/final/publication/geoserver_util.py index 016a41b76..82230362b 100644 --- a/tests/asserts/final/publication/geoserver_util.py +++ b/tests/asserts/final/publication/geoserver_util.py @@ -1,6 +1,9 @@ -def is_complete_in_workspace_wms_instance(wms_instance, name): +def is_complete_in_workspace_wms_instance(wms_instance, name, *, validate_metadata_url): assert wms_instance.contents assert name in wms_instance.contents wms_layer = wms_instance.contents[name] for style_name, style_values in wms_layer.styles.items(): assert 'legend' in style_values, f'style_name={style_name}, style_values={style_values}' + if validate_metadata_url: + assert len(wms_layer.metadataUrls) == 1 + assert wms_layer.metadataUrls[0]['url'].startswith('http://localhost:3080/record/xml/m-') diff --git a/tests/static_data/single_publication/layers_test.py b/tests/static_data/single_publication/layers_test.py index 209286d18..9d8de3e28 100644 --- a/tests/static_data/single_publication/layers_test.py +++ b/tests/static_data/single_publication/layers_test.py @@ -69,7 +69,7 @@ def test_geoserver_workspace(workspace, publ_type, publication): headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0]) publ_asserts.geoserver_proxy.is_complete_in_workspace_wms_1_3_0(workspace, publ_type, publication, headers) - publ_asserts.geoserver_proxy.workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, publication) + publ_asserts.geoserver_proxy.workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, publication, headers) @pytest.mark.parametrize('workspace, publ_type, publication', data.LIST_LAYERS)