From 758df6dae6491b7408b49a6331a9902238891440 Mon Sep 17 00:00:00 2001 From: Jiri Kozel Date: Thu, 8 Apr 2021 10:03:48 +0200 Subject: [PATCH] Migrate layer metadata records: Add LAYERS URL parameter Part of #302 --- CHANGELOG.md | 1 + src/layman/upgrade/__init__.py | 1 + src/layman/upgrade/upgrade_v1_12.py | 44 ++- src/layman/upgrade/upgrade_v1_12_test.py | 94 ++++++ .../upgrade_v1_12_test_layer_metadata.xml | 275 ++++++++++++++++++ 5 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd3301cc..67d104891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Migrations and checks - [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for full-text filtering (install [unaccent](https://www.postgresql.org/docs/10/unaccent.html), create immutable `my_unaccent` function, index unaccented `title` column in `publications` table). - [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for ordering by last change (create and fill column `updated_at` in `publications` table). +- [#302](https://github.com/jirik/layman/issues/302) Add URL parameter `LAYERS` to metadata properties [wms_url](doc/metadata.md#wms_url) and [wfs_url](doc/metadata.md#wfs_url) in existing metadata record of each layer. This non-standard parameter holds name of the layer at given WMS/WFS. ### Changes - [#257](https://github.com/jirik/layman/issues/257) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Maps](doc/rest.md#get-maps) can filter and reorder results according to new query parameters. diff --git a/src/layman/upgrade/__init__.py b/src/layman/upgrade/__init__.py index a87c8bdf2..10f98bdc7 100644 --- a/src/layman/upgrade/__init__.py +++ b/src/layman/upgrade/__init__.py @@ -22,6 +22,7 @@ ]), ((1, 12, 0), [upgrade_v1_12.adjust_prime_db_schema_for_fulltext_search, upgrade_v1_12.adjust_prime_db_schema_for_last_change_search, + upgrade_v1_12.migrate_layer_metadata, ]) ] diff --git a/src/layman/upgrade/upgrade_v1_12.py b/src/layman/upgrade/upgrade_v1_12.py index 1c944da61..7b4c28294 100644 --- a/src/layman/upgrade/upgrade_v1_12.py +++ b/src/layman/upgrade/upgrade_v1_12.py @@ -1,8 +1,16 @@ -import os import datetime +import logging +import os +import time +import requests from layman import settings from layman.common.prime_db_schema import util as db_util +from layman.layer import LAYER_TYPE +from layman.layer.geoserver import wms +from layman.layer.micka import csw as layer_csw + +logger = logging.getLogger(__name__) db_schema = settings.LAYMAN_PRIME_SCHEMA @@ -52,3 +60,37 @@ def adjust_prime_db_schema_for_last_change_search(): statement = f'ALTER TABLE {db_schema}.publications ALTER COLUMN updated_at SET NOT NULL;' db_util.run_statement(statement) + + +def migrate_layer_metadata(workspace_filter=None): + logger.info(f' Starting - migrate layer metadata records') + query = f''' + select w.name, + p.name + from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace + where p.type = %s + ''' + params = (LAYER_TYPE,) + if workspace_filter: + query = query + ' AND w.name = %s' + params = params + (workspace_filter,) + publications = db_util.run_query(query, params) + for (workspace, layer) in publications: + logger.info(f' Migrate layer {workspace}.{layer}') + try: + muuid = layer_csw.patch_layer(workspace, layer, ['wms_url', 'wfs_url'], + create_if_not_exists=False, timeout=2) + if not muuid: + logger.warning(f' Metadata record of layer was not migrated, because the record does not exist.') + except requests.exceptions.ReadTimeout: + md_props = list(layer_csw.get_metadata_comparison(workspace, layer).values()) + md_wms_url = md_props[0]['wms_url'] if md_props else None + base_wms_url = wms.add_capabilities_params_to_url(wms.get_wms_url(workspace, external_url=True)) + exp_wms_url = f"{base_wms_url}?LAYERS={layer}" + if md_wms_url != exp_wms_url: + logger.exception( + f' WMS URL was not migrated (should be {exp_wms_url}, but is {md_wms_url})!') + time.sleep(0.5) + + logger.info(f' DONE - migrate layer metadata records') diff --git a/src/layman/upgrade/upgrade_v1_12_test.py b/src/layman/upgrade/upgrade_v1_12_test.py index 8b7f8210b..525427a9a 100644 --- a/src/layman/upgrade/upgrade_v1_12_test.py +++ b/src/layman/upgrade/upgrade_v1_12_test.py @@ -3,12 +3,63 @@ import pytest from layman import app, settings +from layman.common import geoserver as gs_common +from layman.common.filesystem import uuid as uuid_common +from layman.common.micka import util as micka_util from layman.common.prime_db_schema import util as db_util +from layman.layer import geoserver as gs_layer, NO_STYLE_DEF, db +from layman.layer.geoserver import wms +from layman.layer.prime_db_schema import table as prime_db_schema_table +from layman.uuid import generate_uuid from . import upgrade_v1_12 db_schema = settings.LAYMAN_PRIME_SCHEMA +@pytest.fixture() +def ensure_layer(): + def ensure_layer_internal(workspace, layer): + access_rights = {'read': [settings.RIGHTS_EVERYONE_ROLE], 'write': [settings.RIGHTS_EVERYONE_ROLE], } + with app.app_context(): + uuid_str = generate_uuid() + prime_db_schema_table.post_layer(workspace, + layer, + access_rights, + layer, + uuid_str, + None, + NO_STYLE_DEF, + ) + file_path = '/code/sample/layman.layer/small_layer.geojson' + uuid_common.assign_publication_uuid('layman.layer', workspace, layer, uuid_str=uuid_str) + db.ensure_workspace(workspace) + db.import_layer_vector_file(workspace, layer, file_path, None) + # wfs + created = gs_common.ensure_workspace(workspace, settings.LAYMAN_GS_AUTH) + if created: + gs_common.create_db_store(workspace, settings.LAYMAN_GS_AUTH, db_schema=workspace) + gs_layer.publish_layer_from_db(workspace, layer, layer, layer, None, workspace) + # wms + geoserver_workspace = wms.get_geoserver_workspace(workspace) + created = gs_common.ensure_workspace(geoserver_workspace, settings.LAYMAN_GS_AUTH) + if created: + gs_common.create_db_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, db_schema=workspace) + gs_layer.publish_layer_from_db(workspace, layer, layer, layer, None, geoserver_workspace) + + md_path = '/code/src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml' + with open(md_path, 'r') as template_file: + md_template = template_file.read() + record = md_template.format(uuid=uuid_str, + md_date_stamp=datetime.date.today().strftime('%Y-%m-%d'), + publication_date=datetime.date.today().strftime('%Y-%m-%d'), + workspace=workspace, + layer=layer, + ) + micka_util.soap_insert_record(record, is_public=True) + + yield ensure_layer_internal + + @pytest.mark.usefixtures('ensure_layman') def test_adjust_prime_db_schema_for_last_change_search(): workspace = 'test_adjust_prime_db_schema_for_last_change_search_workspace' @@ -53,3 +104,46 @@ def test_adjust_prime_db_schema_for_last_change_search(): process_client.delete_workspace_layer(workspace, layer) process_client.delete_workspace_map(workspace, map) + + +@pytest.mark.usefixtures('ensure_layman') +def test_migrate_layer_metadata(ensure_layer): + def assert_md_keys(layer_info): + for key in ['comparison_url', 'csw_url', 'identifier', 'record_url']: + assert key in layer_info['metadata'] + + def assert_csw_value(md_comparison, prop_key, exp_value): + csw_prefix = f"http://localhost:3080/csw" + csw_src_key = process_client.get_source_key_from_metadata_comparison(md_comparison, csw_prefix) + assert csw_src_key is not None + md_props = md_comparison['metadata_properties'] + assert md_props[prop_key]['equal'] is True + assert md_props[prop_key]['equal_or_null'] is True + assert md_props[prop_key]['values'][csw_src_key] == exp_value + + workspace = 'test_migrate_layer_metadata_workspace' + layer = 'test_migrate_layer_metadata_layer' + ensure_layer(workspace, layer) + + layer_info = process_client.get_workspace_layer(workspace, layer) + assert_md_keys(layer_info) + + md_comparison = process_client.get_workspace_layer_metadata_comparison(workspace, layer) + exp_wms_url = f"http://localhost:8000/geoserver/{workspace}_wms/ows?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0" + assert_csw_value(md_comparison, 'wms_url', exp_wms_url) + exp_wfs_url = f"http://localhost:8000/geoserver/{workspace}/wfs?SERVICE=WFS&REQUEST=GetCapabilities&VERSION=2.0.0" + assert_csw_value(md_comparison, 'wfs_url', exp_wfs_url) + + with app.app_context(): + upgrade_v1_12.migrate_layer_metadata(workspace) + + layer_info = process_client.get_workspace_layer(workspace, layer) + assert_md_keys(layer_info) + + md_comparison = process_client.get_workspace_layer_metadata_comparison(workspace, layer) + exp_wms_url = f"{exp_wms_url}&LAYERS={layer}" + assert_csw_value(md_comparison, 'wms_url', exp_wms_url) + exp_wfs_url = f"{exp_wfs_url}&LAYERS={layer}" + assert_csw_value(md_comparison, 'wfs_url', exp_wfs_url) + + process_client.delete_workspace_layer(workspace, layer) diff --git a/src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml b/src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml new file mode 100644 index 000000000..a7ebb5e49 --- /dev/null +++ b/src/layman/upgrade/upgrade_v1_12_test_layer_metadata.xml @@ -0,0 +1,275 @@ + + + m-{uuid} + + + eng + + + utf-8 + + + dataset + + + + + pointOfContact + + + + + {md_date_stamp} + + + ISO 19115/INSPIRE_TG2/CZ4 + + + 2003/cor.1/2006 + + + + + + + EPSG:4326 + + + + + + + + + + + EPSG:3857 + + + + + + + + + + + EPSG:5514 + + + + + + + + + + + {layer} + + + + + {publication_date} + + + publication + + + + + + + {layer} + + + + + + + {layer} + + + + + custodian + + + + + + + http://layman_test_run_1:8000/rest/workspaces/{workspace}/layers/{layer}/thumbnail + + + + + + + + + + + + GEMET - INSPIRE themes, version 1.0 + + + + + 2008-06-01 + + + publication + + + + + + + + + + + otherRestrictions + + + Bez omezení + + + + + + + otherRestrictions + + + Žádné podmínky neplatí + + + + + vector + + + + + + + 10000 + + + + + + + utf8 + + + + + + + 14.114369000000037 + + + 14.126824000000035 + + + 48.96483199999997 + + + 48.970612 + + + + + + + + + + + + + + + http://localhost:8000/geoserver/{workspace}_wms/ows?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0 + + + OGC:WMS-1.3.0-http-get-capabilities + + + download + + + + + + + http://localhost:8000/geoserver/{workspace}/wfs?SERVICE=WFS&REQUEST=GetCapabilities&VERSION=2.0.0 + + + OGC:WFS-2.0.0-http-get-capabilities + + + download + + + + + + + http://layman_test_run_1:8000/rest/workspaces/{workspace}/layers/{layer} + + + WWW:LINK-1.0-http--link + + + information + + + + + + + + + + + + + + + + + + + +