Migrate layer metadata records: Add LAYERS URL parameter

Part of #302

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Migrations and checks
 - [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for full-text filtering (install [unaccent](https://www.postgresql.org/docs/10/unaccent.html), create immutable `my_unaccent` function, index unaccented `title` column in `publications` table).
 - [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for ordering by last change (create and fill column `updated_at` in `publications` table).
+- [#302](https://github.com/jirik/layman/issues/302) Add URL parameter `LAYERS` to metadata properties [wms_url](doc/metadata.md#wms_url) and [wfs_url](doc/metadata.md#wfs_url) in existing metadata record of each layer. This non-standard parameter holds name of the layer at given WMS/WFS.
 
 ### Changes
 - [#257](https://github.com/jirik/layman/issues/257) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Maps](doc/rest.md#get-maps) can filter and reorder results according to new query parameters.

src/layman/upgrade/__init__.py

@@ -22,6 +22,7 @@
                   ]),
     ((1, 12, 0), [upgrade_v1_12.adjust_prime_db_schema_for_fulltext_search,
                   upgrade_v1_12.adjust_prime_db_schema_for_last_change_search,
+                  upgrade_v1_12.migrate_layer_metadata,
                   ])
 ]
 

src/layman/upgrade/upgrade_v1_12.py

@@ -1,8 +1,16 @@
-import os
 import datetime
+import logging
+import os
+import time
+import requests
 
 from layman import settings
 from layman.common.prime_db_schema import util as db_util
+from layman.layer import LAYER_TYPE
+from layman.layer.geoserver import wms
+from layman.layer.micka import csw as layer_csw
+
+logger = logging.getLogger(__name__)
 
 db_schema = settings.LAYMAN_PRIME_SCHEMA
 
@@ -52,3 +60,37 @@ def adjust_prime_db_schema_for_last_change_search():
 
     statement = f'ALTER TABLE {db_schema}.publications ALTER COLUMN updated_at SET NOT NULL;'
     db_util.run_statement(statement)
+
+
+def migrate_layer_metadata(workspace_filter=None):
+    logger.info(f'    Starting - migrate layer metadata records')
+    query = f'''
+    select  w.name,
+            p.name
+    from {db_schema}.publications p inner join
+         {db_schema}.workspaces w on w.id = p.id_workspace
+    where p.type = %s
+    '''
+    params = (LAYER_TYPE,)
+    if workspace_filter:
+        query = query + '  AND w.name = %s'
+        params = params + (workspace_filter,)
+    publications = db_util.run_query(query, params)
+    for (workspace, layer) in publications:
+        logger.info(f'      Migrate layer {workspace}.{layer}')
+        try:
+            muuid = layer_csw.patch_layer(workspace, layer, ['wms_url', 'wfs_url'],
+                                          create_if_not_exists=False, timeout=2)
+            if not muuid:
+                logger.warning(f'        Metadata record of layer was not migrated, because the record does not exist.')
+        except requests.exceptions.ReadTimeout:
+            md_props = list(layer_csw.get_metadata_comparison(workspace, layer).values())
+            md_wms_url = md_props[0]['wms_url'] if md_props else None
+            base_wms_url = wms.add_capabilities_params_to_url(wms.get_wms_url(workspace, external_url=True))
+            exp_wms_url = f"{base_wms_url}?LAYERS={layer}"
+            if md_wms_url != exp_wms_url:
+                logger.exception(
+                    f'        WMS URL was not migrated (should be {exp_wms_url}, but is {md_wms_url})!')
+        time.sleep(0.5)
+
+    logger.info(f'    DONE - migrate layer metadata records')

src/layman/upgrade/upgrade_v1_12_test.py

@@ -3,12 +3,63 @@
 import pytest
 
 from layman import app, settings
+from layman.common import geoserver as gs_common
+from layman.common.filesystem import uuid as uuid_common
+from layman.common.micka import util as micka_util
 from layman.common.prime_db_schema import util as db_util
+from layman.layer import geoserver as gs_layer, NO_STYLE_DEF, db
+from layman.layer.geoserver import wms
+from layman.layer.prime_db_schema import table as prime_db_schema_table
+from layman.uuid import generate_uuid
 from . import upgrade_v1_12
 
 db_schema = settings.LAYMAN_PRIME_SCHEMA
 
 
+@pytest.fixture()
+def ensure_layer():
+    def ensure_layer_internal(workspace, layer):
+        access_rights = {'read': [settings.RIGHTS_EVERYONE_ROLE], 'write': [settings.RIGHTS_EVERYONE_ROLE], }
+        with app.app_context():
+            uuid_str = generate_uuid()
+            prime_db_schema_table.post_layer(workspace,
+                                             layer,
+                                             access_rights,
+                                             layer,
+                                             uuid_str,
+                                             None,
+                                             NO_STYLE_DEF,
+                                             )
+            file_path = '/code/sample/layman.layer/small_layer.geojson'
+            uuid_common.assign_publication_uuid('layman.layer', workspace, layer, uuid_str=uuid_str)
+            db.ensure_workspace(workspace)
+            db.import_layer_vector_file(workspace, layer, file_path, None)
+            # wfs
+            created = gs_common.ensure_workspace(workspace, settings.LAYMAN_GS_AUTH)
+            if created:
+                gs_common.create_db_store(workspace, settings.LAYMAN_GS_AUTH, db_schema=workspace)
+            gs_layer.publish_layer_from_db(workspace, layer, layer, layer, None, workspace)
+            # wms
+            geoserver_workspace = wms.get_geoserver_workspace(workspace)
+            created = gs_common.ensure_workspace(geoserver_workspace, settings.LAYMAN_GS_AUTH)
+            if created:
+                gs_common.create_db_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, db_schema=workspace)
+            gs_layer.publish_layer_from_db(workspace, layer, layer, layer, None, geoserver_workspace)
+
+            md_path = '/code/src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml'
+            with open(md_path, 'r') as template_file:
+                md_template = template_file.read()
+            record = md_template.format(uuid=uuid_str,
+                                        md_date_stamp=datetime.date.today().strftime('%Y-%m-%d'),
+                                        publication_date=datetime.date.today().strftime('%Y-%m-%d'),
+                                        workspace=workspace,
+                                        layer=layer,
+                                        )
+            micka_util.soap_insert_record(record, is_public=True)
+
+    yield ensure_layer_internal
+
+
 @pytest.mark.usefixtures('ensure_layman')
 def test_adjust_prime_db_schema_for_last_change_search():
     workspace = 'test_adjust_prime_db_schema_for_last_change_search_workspace'
@@ -53,3 +104,46 @@ def test_adjust_prime_db_schema_for_last_change_search():
 
     process_client.delete_workspace_layer(workspace, layer)
     process_client.delete_workspace_map(workspace, map)
+
+
+@pytest.mark.usefixtures('ensure_layman')
+def test_migrate_layer_metadata(ensure_layer):
+    def assert_md_keys(layer_info):
+        for key in ['comparison_url', 'csw_url', 'identifier', 'record_url']:
+            assert key in layer_info['metadata']
+
+    def assert_csw_value(md_comparison, prop_key, exp_value):
+        csw_prefix = f"http://localhost:3080/csw"
+        csw_src_key = process_client.get_source_key_from_metadata_comparison(md_comparison, csw_prefix)
+        assert csw_src_key is not None
+        md_props = md_comparison['metadata_properties']
+        assert md_props[prop_key]['equal'] is True
+        assert md_props[prop_key]['equal_or_null'] is True
+        assert md_props[prop_key]['values'][csw_src_key] == exp_value
+
+    workspace = 'test_migrate_layer_metadata_workspace'
+    layer = 'test_migrate_layer_metadata_layer'
+    ensure_layer(workspace, layer)
+
+    layer_info = process_client.get_workspace_layer(workspace, layer)
+    assert_md_keys(layer_info)
+
+    md_comparison = process_client.get_workspace_layer_metadata_comparison(workspace, layer)
+    exp_wms_url = f"http://localhost:8000/geoserver/{workspace}_wms/ows?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0"
+    assert_csw_value(md_comparison, 'wms_url', exp_wms_url)
+    exp_wfs_url = f"http://localhost:8000/geoserver/{workspace}/wfs?SERVICE=WFS&REQUEST=GetCapabilities&VERSION=2.0.0"
+    assert_csw_value(md_comparison, 'wfs_url', exp_wfs_url)
+
+    with app.app_context():
+        upgrade_v1_12.migrate_layer_metadata(workspace)
+
+    layer_info = process_client.get_workspace_layer(workspace, layer)
+    assert_md_keys(layer_info)
+
+    md_comparison = process_client.get_workspace_layer_metadata_comparison(workspace, layer)
+    exp_wms_url = f"{exp_wms_url}&LAYERS={layer}"
+    assert_csw_value(md_comparison, 'wms_url', exp_wms_url)
+    exp_wfs_url = f"{exp_wfs_url}&LAYERS={layer}"
+    assert_csw_value(md_comparison, 'wfs_url', exp_wfs_url)
+
+    process_client.delete_workspace_layer(workspace, layer)