Add MetadataURL to existing layers definition on GS

LayerManager · May 25, 2023 · 3c0383b · 3c0383b
1 parent b175373
commit 3c0383b
Show file tree

Hide file tree

Showing 3 changed files with 185 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,7 @@
 - [#528](https://github.com/LayerManager/layman/issues/528) Add new data type `enum_wfs_wms_status` and create new string column `wfs_wms_status` in `publications` table in prime DB schema.
 #### Data migrations
 - [#528](https://github.com/LayerManager/layman/issues/528) Fill column `wfs_wms_status` in `publications` table in prime DB schema. For layers set value `AVAILABLE` if fully available through WFS and WFS, otherwise `NOT_AVAILABLE`. Value is set to `null` for all existing maps. 
+- [#520](https://github.com/LayerManager/layman/issues/520) Set MetadataURL for each layer in WFS and WMS workspace in GeoServer.
 ### Changes
 - [#528](https://github.com/LayerManager/layman/issues/528) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Workspace Layers](doc/rest.md#get-workspace-layers) returns new key `wfs_wms_status`.
 - [#765](https://github.com/LayerManager/layman/issues/765) Remove Liferay from dev stack, use [Wagtail CRX](https://docs.coderedcorp.com/wagtail-crx/) + [Django OAuth Toolkit](https://django-oauth-toolkit.readthedocs.io/en/latest/) as new OAuth2 provider (authorization server).

diff --git a/src/layman/upgrade/upgrade_v1_21.py b/src/layman/upgrade/upgrade_v1_21.py
@@ -1,8 +1,14 @@
+import json
 import logging
+from urllib.parse import urljoin
+import requests
 
 from db import util as db_util
+from geoserver import GS_REST_WORKSPACES, GS_REST_TIMEOUT, util as gs_common_util
 from layman import settings
+from layman.common.micka import util as micka_util
 from layman.layer import LAYER_TYPE, util
+from layman.layer.geoserver import wms, util as gs_util
 from layman.map import MAP_TYPE
 
 logger = logging.getLogger(__name__)
@@ -42,3 +48,75 @@ def adjust_publications_wfs_wms_status():
     statement = f'alter table {DB_SCHEMA}.publications add constraint wfs_wms_status_with_type_check CHECK ' \
                 f'((wfs_wms_status IS NULL AND type = %s) OR (wfs_wms_status IS NOT NULL AND type = %s));'
     db_util.run_statement(statement, (MAP_TYPE, LAYER_TYPE,))
+
+
+def adjust_layer_metadata_url_on_gs():
+    logger.info(f'    Adjust layer MetadataUrl on GeoServer')
+
+    auth = settings.LAYMAN_GS_AUTH
+
+    query = f'''select w.name, p.name, p.geodata_type, p.style_type, p.image_mosaic, p.uuid, PGP_SYM_DECRYPT(p.external_table_uri, p.uuid::text)::json external_table_uri
+    from {DB_SCHEMA}.publications p inner join
+         {DB_SCHEMA}.workspaces w on w.id = p.id_workspace
+    where p.type = %s
+      and p.wfs_wms_status = %s
+    ;'''
+    layers = db_util.run_query(query, (LAYER_TYPE, settings.EnumWfsWmsStatus.AVAILABLE.value))
+
+    for workspace, layer, geodata_type, style_type, image_mosaic, uuid, external_table_uri in layers:
+        logger.info(f'      Layer {workspace}:{layer}')
+        wms_workspace = wms.get_geoserver_workspace(workspace)
+
+        metadata_links = {
+            'metadataLinks':
+                {
+                    "metadataLink": [
+                        {
+                            "type": "application/xml",
+                            "metadataType": "ISO19115:2003",
+                            "content": micka_util.get_metadata_url(uuid, url_type=micka_util.RecordUrlType.XML),
+                        }
+                    ]
+                }
+        }
+
+        if geodata_type == settings.GEODATA_TYPE_RASTER:
+            wms_body = {"coverage": metadata_links}
+            store_name = wms.get_image_mosaic_store_name(layer) if image_mosaic else wms.get_geotiff_store_name(layer)
+            wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/coveragestores/{store_name}/coverages/{layer}')
+        elif geodata_type == settings.GEODATA_TYPE_VECTOR:
+            if style_type == 'sld':
+                wms_body = {"featureType": metadata_links}
+                store_name = gs_util.get_external_db_store_name(layer) if external_table_uri else gs_common_util.DEFAULT_DB_STORE_NAME
+                wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/datastores/{store_name}/featuretypes/{layer}')
+            elif style_type == 'qml':
+                wms_layer = gs_common_util.get_wms_layer(wms_workspace, layer, auth=auth)
+                wms_layer = {**wms_layer, **metadata_links}
+                wms_body = {"wmsLayer": wms_layer}
+                wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/wmslayers/{layer}')
+            else:
+                raise NotImplementedError(f"Unknown style type: {style_type}")
+
+            # WFS
+            wfs_body = {"featureType": metadata_links}
+            wfs_store = gs_util.get_external_db_store_name(layer) if external_table_uri else gs_common_util.DEFAULT_DB_STORE_NAME
+            response = requests.put(
+                urljoin(GS_REST_WORKSPACES, f'{workspace}/datastores/{wfs_store}/featuretypes/{layer}'),
+                data=json.dumps(wfs_body),
+                headers=gs_common_util.headers_json,
+                auth=auth,
+                timeout=GS_REST_TIMEOUT,
+            )
+            response.raise_for_status()
+        else:
+            raise NotImplementedError(f"Unknown geodata type: {geodata_type}")
+
+        # WMS
+        response = requests.put(
+            wms_url,
+            data=json.dumps(wms_body),
+            headers=gs_common_util.headers_json,
+            auth=auth,
+            timeout=GS_REST_TIMEOUT,
+        )
+        response.raise_for_status()
diff --git a/src/layman/upgrade/upgrade_v1_21_test.py b/src/layman/upgrade/upgrade_v1_21_test.py
@@ -1,8 +1,16 @@
+import json
+from urllib.parse import urljoin
+import requests
 import pytest
 
 from db import util as db_util
-from layman import app, settings
-from test_tools import process_client
+from geoserver import GS_REST_WORKSPACES, GS_REST_TIMEOUT, util as gs_common_util
+from layman import app, settings, util as layman_util
+from layman.layer.geoserver import wms, util as gs_util
+from test_tools import process_client, util as test_util
+from tests import Publication
+from tests.asserts.final.publication import util as asserts_util
+from tests.dynamic_data.publications import common_publications
 from . import upgrade_v1_21
 
 DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA
@@ -79,3 +87,99 @@ def test_wfs_wms_status(publ_type, publ_name, rest_args, exp_wfs_wms_status):
         assert wfs_wms_status is None, f'publication={publ_type}:{workspace}.{publ_name}, wfs_wms_status={wfs_wms_status}'
 
     process_client.delete_workspace_publication(publ_type, workspace, publ_name)
+
+
+@pytest.mark.usefixtures('ensure_layman')
+def test_adjust_layer_metadata_url_on_gs():
+    workspace = 'test_adjust_metadata_url_workspace'
+    auth = settings.LAYMAN_GS_AUTH
+
+    layers = [
+        ('vector_sld', common_publications.LAYER_VECTOR_SLD.definition),
+        ('vector_qml', common_publications.LAYER_VECTOR_QML.definition),
+        ('raster', common_publications.LAYER_RASTER.definition),
+    ]
+
+    for layer, layer_def in layers:
+        process_client.publish_workspace_publication(process_client.LAYER_TYPE, workspace, layer, **layer_def)
+        with app.app_context():
+            publ_info = layman_util.get_publication_info(workspace, process_client.LAYER_TYPE, layer,
+                                                         context={'keys': {'geodata_type', 'style_type', 'image_mosaic',
+                                                                           'original_data_source'}})
+        wms_workspace = wms.get_geoserver_workspace(workspace)
+
+        if publ_info['geodata_type'] == settings.GEODATA_TYPE_RASTER:
+            store_name = wms.get_image_mosaic_store_name(layer) if publ_info['image_mosaic'] else wms.get_geotiff_store_name(layer)
+            wms_coverage = gs_common_util.get_coverage(wms_workspace, store_name, layer, gs_rest_workspaces=GS_REST_WORKSPACES)
+            wms_coverage['metadataLinks'] = {}
+            wms_body = {"coverage": wms_coverage}
+            wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/coveragestores/{store_name}/coverages/{layer}')
+        elif publ_info['geodata_type'] == settings.GEODATA_TYPE_VECTOR:
+            if publ_info['_style_type'] == 'sld':
+                store_name = gs_util.get_external_db_store_name(layer) if publ_info[
+                    'original_data_source'] == settings.EnumOriginalDataSource.TABLE.value else gs_common_util.DEFAULT_DB_STORE_NAME
+                wms_feature_type = gs_common_util.get_feature_type(wms_workspace, store_name, layer, gs_rest_workspaces=GS_REST_WORKSPACES)
+                wms_feature_type['metadataLinks'] = {}
+                wms_body = {"featureType": wms_feature_type}
+                wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/datastores/{store_name}/featuretypes/{layer}')
+            elif publ_info['_style_type'] == 'qml':
+                wms_layer = gs_common_util.get_wms_layer(wms_workspace, layer, auth=auth)
+                wms_layer['metadataLinks'] = {}
+                wms_body = {"wmsLayer": wms_layer}
+                wms_url = urljoin(GS_REST_WORKSPACES, f'{wms_workspace}/wmslayers/{layer}')
+            else:
+                raise NotImplementedError(f"Unknown style type: {publ_info['_style_type']}")
+
+            # WFS
+            wfs_store = gs_util.get_external_db_store_name(layer) if publ_info[
+                'original_data_source'] == settings.EnumOriginalDataSource.TABLE.value else gs_common_util.DEFAULT_DB_STORE_NAME
+
+            wfs_inner_body = gs_common_util.get_feature_type(workspace, wfs_store, layer, gs_rest_workspaces=GS_REST_WORKSPACES)
+            wfs_inner_body['metadataLinks'] = {}
+            wfs_body = {"featureType": wfs_inner_body}
+            response = requests.put(
+                urljoin(GS_REST_WORKSPACES, f'{workspace}/datastores/{wfs_store}/featuretypes/{layer}'),
+                data=json.dumps(wfs_body),
+                headers=gs_common_util.headers_json,
+                auth=auth,
+                timeout=GS_REST_TIMEOUT,
+            )
+            response.raise_for_status()
+
+            # Validate MetadataLinks emptiness
+            with app.app_context():
+                wfs_url = test_util.url_for('geoserver_proxy_bp.proxy',
+                                            subpath=f'{workspace}/ows')
+            wfs_inst = gs_util.wfs_proxy(wfs_url, version='2.0.0')
+            wfs_layer = wfs_inst.contents[f'{workspace}:{layer}']
+            assert len(wfs_layer.metadataUrls) == 0, f'layer={layer}, wfs_layer.metadataUrls={wfs_layer.metadataUrls}'
+        else:
+            raise NotImplementedError(f"Unknown geodata type: {publ_info['geodata_type']}")
+
+        # WMS
+        response = requests.put(
+            wms_url,
+            data=json.dumps(wms_body),
+            headers=gs_common_util.headers_json,
+            auth=auth,
+            timeout=GS_REST_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        # Validate MetadataLinks emptiness
+        with app.app_context():
+            wms_url = test_util.url_for('geoserver_proxy_bp.proxy',
+                                        subpath=f'{workspace}{settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX}/ows')
+        wms_inst = gs_util.wms_proxy(wms_url, version='1.3.0')
+        wms_layer = wms_inst.contents[layer]
+        assert len(wms_layer.metadataUrls) == 0, f'layer={layer}, wms_layer.metadataUrls={wms_layer.metadataUrls}'
+
+    with app.app_context():
+        upgrade_v1_21.adjust_layer_metadata_url_on_gs()
+
+    for layer, _ in layers:
+        asserts_util.is_publication_valid_and_complete(Publication(workspace=workspace,
+                                                                   type=process_client.LAYER_TYPE,
+                                                                   name=layer))
+
+    process_client.delete_workspace_layers(workspace)