Add MetadataURL to layer definition on GS

CHANGELOG.md

@@ -47,6 +47,7 @@
   - flask 2.2.2 -> 2.3.2
   - redis 4.5.1 -> 4.5.4
   - owslib 0.27.2 -> 0.28.1
+- [#520](https://github.com/LayerManager/layman/issues/520) WFS in version `2.0.0` and WMS in version `1.3.0` includes also MetadataURL for each layer with url of Micka record.
 - [#833](https://github.com/LayerManager/layman/issues/833) Make Timgen WMS requests more robust (handle WML errors, delayed retry, add timestamp to each request).
 - [#815](https://github.com/LayerManager/layman/issues/815) Propagate [`LAYMAN_PROXY_SERVER_NAME`](doc/env-settings.md#LAYMAN_PROXY_SERVER_NAME) value to GeoServer environment variable [GEOSERVER_CSRF_WHITELIST](https://docs.geoserver.org/latest/en/user/security/webadmin/csrf.html).
 - [#847](https://github.com/LayerManager/layman/issues/847) Fix publishing external table layers with `@` in the username or the password.

src/geoserver/util.py

@@ -273,7 +273,7 @@ def patch_feature_type(geoserver_workspace, feature_type_name, store_name=None,
     response.raise_for_status()
 
 
-def post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, auth, *, lat_lon_bbox, table_name, store_name=None):
+def post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, auth, *, lat_lon_bbox, table_name, metadata_url, store_name=None, ):
     store_name = store_name or DEFAULT_DB_STORE_NAME
     keywords = [
         "features",
@@ -298,6 +298,15 @@ def post_feature_type(geoserver_workspace, layername, description, title, bbox,
         },
         'nativeBoundingBox': bbox_to_dict(bbox, crs),
         'latLonBoundingBox': bbox_to_dict(lat_lon_bbox, 'CRS:84'),
+        'metadataLinks': {
+            "metadataLink": [
+                {
+                    "type": "application/xml",
+                    "metadataType": "ISO19115:2003",
+                    "content": metadata_url,
+                }
+            ]
+        }
     }
     response = requests.post(urljoin(GS_REST_WORKSPACES,
                                      f'{geoserver_workspace}/datastores/{store_name}/featuretypes/'),
@@ -615,7 +624,8 @@ def delete_coverage_store(geoserver_workspace, auth, name):
         response.raise_for_status()
 
 
-def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, description, bbox, crs, *, lat_lon_bbox, enable_time_dimension=False):
+def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, description, bbox, crs, *, lat_lon_bbox, metadata_url,
+                     enable_time_dimension=False):
     keywords = [
         "features",
         layer,
@@ -641,6 +651,15 @@ def publish_coverage(geoserver_workspace, auth, coverage_store, layer, title, de
                 "name": f"{geoserver_workspace}:{coverage_store}"
             },
             "title": title,
+            'metadataLinks': {
+                "metadataLink": [
+                    {
+                        "type": "application/xml",
+                        "metadataType": "ISO19115:2003",
+                        "content": metadata_url,
+                    }
+                ]
+            }
         }
     }
     if enable_time_dimension:
@@ -755,7 +774,7 @@ def patch_wms_layer(geoserver_workspace, layer, *, auth, bbox=None, title=None,
     response.raise_for_status()
 
 
-def post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, auth, *, lat_lon_bbox):
+def post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, auth, *, lat_lon_bbox, metadata_url):
     keywords = [
         "features",
         layer,
@@ -780,6 +799,15 @@ def post_wms_layer(geoserver_workspace, layer, store_name, title, description, b
         },
         'nativeBoundingBox': bbox_to_dict(bbox, crs),
         'latLonBoundingBox': bbox_to_dict(lat_lon_bbox, 'CRS:84'),
+        'metadataLinks': {
+            "metadataLink": [
+                {
+                    "type": "application/xml",
+                    "metadataType": "ISO19115:2003",
+                    "content": metadata_url,
+                }
+            ]
+        }
     }
     response = requests.post(urljoin(GS_REST_WORKSPACES,
                                      geoserver_workspace + '/wmslayers/'),

src/layman/common/micka/util.py

@@ -1,3 +1,4 @@
+from enum import Enum
 import os
 import time
 from io import BytesIO
@@ -19,6 +20,11 @@
 logger = logging.getLogger(__name__)
 
 
+class RecordUrlType(Enum):
+    BASIC = 'basic'
+    XML = 'xml'
+
+
 for k, v in NAMESPACES.items():
     ET.register_namespace(k, v)
 
@@ -27,10 +33,10 @@ def get_metadata_uuid(uuid):
     return f"m-{uuid}" if uuid is not None else None
 
 
-def get_metadata_url(uuid):
+def get_metadata_url(uuid, *, url_type: RecordUrlType):
     muuid = get_metadata_uuid(uuid)
     server_url = settings.CSW_PROXY_URL[:-3]
-    result = f'{server_url}record/basic/{muuid}'
+    result = f'{server_url}record/{url_type.value}/{muuid}'
     return result
 
 

src/layman/layer/geoserver/__init__.py

@@ -123,14 +123,14 @@ def get_layer_native_bbox(workspace, layer):
     return gs_util.bbox_to_dict(bbox, crs)
 
 
-def publish_layer_from_db(workspace, layername, description, title, *, crs, table_name, geoserver_workspace=None, store_name=None):
+def publish_layer_from_db(workspace, layername, description, title, *, crs, table_name, metadata_url, geoserver_workspace=None, store_name=None):
     geoserver_workspace = geoserver_workspace or workspace
     bbox = get_layer_bbox(workspace, layername)
     lat_lon_bbox = bbox_util.transform(bbox, crs, crs_def.EPSG_4326)
-    gs_util.post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox, table_name=table_name, store_name=store_name)
+    gs_util.post_feature_type(geoserver_workspace, layername, description, title, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox, table_name=table_name, metadata_url=metadata_url, store_name=store_name)
 
 
-def publish_layer_from_qgis(workspace, layer, description, title, *, geoserver_workspace=None):
+def publish_layer_from_qgis(workspace, layer, description, title, *, metadata_url, geoserver_workspace=None):
     geoserver_workspace = geoserver_workspace or workspace
     store_name = wms.get_qgis_store_name(layer)
     info = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['wms', 'native_crs', ]})
@@ -142,7 +142,8 @@ def publish_layer_from_qgis(workspace, layer, description, title, *, geoserver_w
                              layer_capabilities_url)
     bbox = get_layer_bbox(workspace, layer)
     lat_lon_bbox = bbox_util.transform(bbox, crs, crs_def.EPSG_4326)
-    gs_util.post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, settings.LAYMAN_GS_AUTH, lat_lon_bbox=lat_lon_bbox)
+    gs_util.post_wms_layer(geoserver_workspace, layer, store_name, title, description, bbox, crs, settings.LAYMAN_GS_AUTH,
+                           lat_lon_bbox=lat_lon_bbox, metadata_url=metadata_url)
 
 
 def get_usernames():

src/layman/layer/geoserver/tasks.py

@@ -7,6 +7,7 @@
 from layman.celery import AbortedException
 from layman import celery_app, settings, util as layman_util
 from layman.common import empty_method_returns_true, bbox as bbox_util
+from layman.common.micka import util as micka_util
 from layman.http import LaymanError
 from . import wms, wfs, sld
 from .. import geoserver, LAYER_TYPE
@@ -38,7 +39,7 @@ def refresh_wms(
         original_data_source=settings.EnumOriginalDataSource.FILE.value,
 ):
     info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': [
-        'file', 'geodata_type', 'native_bounding_box', 'native_crs', 'table_uri'
+        'file', 'geodata_type', 'native_bounding_box', 'native_crs', 'table_uri', 'uuid'
     ]})
     geodata_type = info['geodata_type']
     crs = info['native_crs']
@@ -47,6 +48,7 @@ def refresh_wms(
     assert title is not None
     geoserver_workspace = wms.get_geoserver_workspace(workspace)
     geoserver.ensure_workspace(workspace)
+    metadata_url = micka_util.get_metadata_url(info['uuid'], url_type=micka_util.RecordUrlType.XML)
 
     if self.is_aborted():
         raise AbortedException
@@ -67,6 +69,7 @@ def refresh_wms(
                                             title,
                                             crs=crs,
                                             table_name=table_name,
+                                            metadata_url=metadata_url,
                                             geoserver_workspace=geoserver_workspace,
                                             store_name=store_name,
                                             )
@@ -75,6 +78,7 @@ def refresh_wms(
                                               layername,
                                               description,
                                               title,
+                                              metadata_url=metadata_url,
                                               geoserver_workspace=geoserver_workspace,
                                               )
     elif geodata_type == settings.GEODATA_TYPE_RASTER:
@@ -102,7 +106,7 @@ def refresh_wms(
             enable_time_dimension = True
         gs_util.create_coverage_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, coverage_store_name, source_file_or_dir, coverage_type=coverage_type)
         gs_util.publish_coverage(geoserver_workspace, settings.LAYMAN_GS_AUTH, coverage_store_name, layername, title,
-                                 description, bbox, crs, lat_lon_bbox=lat_lon_bbox, enable_time_dimension=enable_time_dimension)
+                                 description, bbox, crs, lat_lon_bbox=lat_lon_bbox, metadata_url=metadata_url, enable_time_dimension=enable_time_dimension)
     else:
         raise NotImplementedError(f"Unknown geodata type: {geodata_type}")
 
@@ -139,7 +143,7 @@ def refresh_wfs(
         access_rights=None,
         original_data_source=settings.EnumOriginalDataSource.FILE.value,
 ):
-    info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['geodata_type', 'native_crs', 'table_uri']})
+    info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['geodata_type', 'native_crs', 'table_uri', 'uuid']})
     geodata_type = info['geodata_type']
     if geodata_type == settings.GEODATA_TYPE_RASTER:
         return
@@ -161,7 +165,8 @@ def refresh_wfs(
                                                         layer=layername,
                                                         table_uri=table_uri,
                                                         )
-    geoserver.publish_layer_from_db(workspace, layername, description, title, crs=crs, table_name=table_name, store_name=store_name)
+    metadata_url = micka_util.get_metadata_url(info['uuid'], url_type=micka_util.RecordUrlType.XML)
+    geoserver.publish_layer_from_db(workspace, layername, description, title, crs=crs, table_name=table_name, metadata_url=metadata_url, store_name=store_name)
     geoserver.set_security_rules(workspace, layername, access_rights, settings.LAYMAN_GS_AUTH, workspace)
     wfs.clear_cache(workspace)
 

src/layman/layer/micka/csw.py

@@ -49,7 +49,7 @@ def get_layer_info(workspace, layername):
             'metadata': {
                 'identifier': muuid,
                 'csw_url': settings.CSW_PROXY_URL,
-                'record_url': common_util.get_metadata_url(uuid),
+                'record_url': common_util.get_metadata_url(uuid, url_type=common_util.RecordUrlType.BASIC),
                 'comparison_url': url_for('rest_workspace_layer_metadata_comparison.get', workspace=workspace, layername=layername),
             }
         }

src/layman/map/micka/csw.py

@@ -45,7 +45,7 @@ def get_map_info(workspace, mapname):
             'metadata': {
                 'identifier': muuid,
                 'csw_url': settings.CSW_PROXY_URL,
-                'record_url': common_util.get_metadata_url(uuid),
+                'record_url': common_util.get_metadata_url(uuid, url_type=common_util.RecordUrlType.BASIC),
                 'comparison_url': url_for('rest_workspace_map_metadata_comparison.get', workspace=workspace, mapname=mapname),
             }
         }

src/layman/upgrade/upgrade_v1_17_test.py

@@ -6,6 +6,7 @@
 from layman import app, settings, util as layman_util
 from layman.common.prime_db_schema import publications as prime_db_schema_publications
 from layman.common.filesystem import uuid as uuid_common
+from layman.common.micka import util as micka_util
 from layman.layer import LAYER_TYPE, STYLE_TYPES_DEF, db, geoserver, qgis
 from layman.layer.db import table
 from layman.layer.geoserver import wms, wfs
@@ -150,13 +151,16 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ):
         geoserver.ensure_workspace(workspace)
         geoserver.ensure_workspace(wms_workspace)
 
+        metadata_url = micka_util.get_metadata_url(uuid_str, url_type=micka_util.RecordUrlType.XML)
+
         # import into GS WFS workspace
         geoserver.publish_layer_from_db(workspace,
                                         layer,
                                         description=layer,
                                         title=layer,
                                         crs=crs,
                                         table_name=table_name,
+                                        metadata_url=metadata_url,
                                         geoserver_workspace=workspace,
                                         )
 
@@ -168,6 +172,7 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ):
                                             title=layer,
                                             crs=crs,
                                             table_name=table_name,
+                                            metadata_url=metadata_url,
                                             geoserver_workspace=wms_workspace,
                                             )
         elif style_type == 'qml':
@@ -195,6 +200,7 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ):
                                               layer,
                                               description=layer,
                                               title=layer,
+                                              metadata_url=metadata_url,
                                               geoserver_workspace=wms_workspace,
                                               )
         for gs_workspace in [workspace, wms.get_geoserver_workspace(workspace)]:

tests/asserts/final/publication/geoserver.py

@@ -129,7 +129,7 @@ def is_complete_in_internal_workspace_wms(workspace, publ_type, name):
     assert publ_type == process_client.LAYER_TYPE
 
     wms_inst = wms.get_wms_proxy(workspace)
-    geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name)
+    geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name, validate_metadata_url=False)
 
 
 def assert_workspace_stores(workspace, *, exp_stores=None, exp_existing_stores=None, exp_deleted_stores=None):

tests/asserts/final/publication/geoserver_proxy.py

@@ -1,5 +1,3 @@
-import requests
-
 from layman import app, settings, util as layman_util
 from layman.layer.geoserver import util as gs_util
 from test_tools import util as test_util, process_client
@@ -12,25 +10,28 @@ def is_complete_in_workspace_wms(workspace, publ_type, name, *, version, headers
     with app.app_context():
         wms_url = test_util.url_for('geoserver_proxy_bp.proxy', subpath=workspace + settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX + '/ows')
     wms_inst = gs_util.wms_proxy(wms_url, version=version, headers=headers)
-    geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name)
+    validate_metadata_url = version != '1.1.1'
+    geoserver_util.is_complete_in_workspace_wms_instance(wms_inst, name, validate_metadata_url=validate_metadata_url)
 
 
 def is_complete_in_workspace_wms_1_3_0(workspace, publ_type, name, headers):
     assert publ_type == process_client.LAYER_TYPE
     is_complete_in_workspace_wms(workspace, publ_type, name, version='1.3.0', headers=headers)
 
 
-def workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, name):
+def workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, name, headers):
     with app.app_context():
         internal_wfs_url = test_util.url_for('geoserver_proxy_bp.proxy', subpath=workspace + '/wfs')
 
     with app.app_context():
         file_info = layman_util.get_publication_info(workspace, publ_type, name, {'keys': ['geodata_type']})
     geodata_type = file_info['geodata_type']
     if geodata_type == settings.GEODATA_TYPE_VECTOR:
-        r_wfs = requests.get(internal_wfs_url, params={
-            'service': 'WFS',
-            'request': 'GetCapabilities',
-            'version': '2.0.0',
-        }, timeout=settings.DEFAULT_CONNECTION_TIMEOUT)
-        assert r_wfs.status_code == 200
+        wfs_inst = gs_util.wfs_proxy(wfs_url=internal_wfs_url, version='2.0.0', headers=headers)
+
+        assert wfs_inst.contents
+        wfs_name = f'{workspace}:{name}'
+        assert wfs_name in wfs_inst.contents
+        wfs_layer = wfs_inst.contents[wfs_name]
+        assert len(wfs_layer.metadataUrls) == 1
+        assert wfs_layer.metadataUrls[0]['url'].startswith('http://localhost:3080/record/xml/m-')

tests/asserts/final/publication/geoserver_util.py

@@ -1,6 +1,9 @@
-def is_complete_in_workspace_wms_instance(wms_instance, name):
+def is_complete_in_workspace_wms_instance(wms_instance, name, *, validate_metadata_url):
     assert wms_instance.contents
     assert name in wms_instance.contents
     wms_layer = wms_instance.contents[name]
     for style_name, style_values in wms_layer.styles.items():
         assert 'legend' in style_values, f'style_name={style_name}, style_values={style_values}'
+    if validate_metadata_url:
+        assert len(wms_layer.metadataUrls) == 1
+        assert wms_layer.metadataUrls[0]['url'].startswith('http://localhost:3080/record/xml/m-')

tests/static_data/single_publication/layers_test.py

@@ -69,7 +69,7 @@ def test_geoserver_workspace(workspace, publ_type, publication):
     headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
 
     publ_asserts.geoserver_proxy.is_complete_in_workspace_wms_1_3_0(workspace, publ_type, publication, headers)
-    publ_asserts.geoserver_proxy.workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, publication)
+    publ_asserts.geoserver_proxy.workspace_wfs_2_0_0_capabilities_available_if_vector(workspace, publ_type, publication, headers)
 
 
 @pytest.mark.parametrize('workspace, publ_type, publication', data.LIST_LAYERS)