diff --git a/CHANGELOG.md b/CHANGELOG.md index de941f839..da31dc9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,10 @@ - Fix: Raise error when more than one main layer file is sent in [POST Workspace Layers](doc/rest.md#post-workspace-layers) or [PATCH Workspace Layer](doc/rest.md#patch-workspace-layer). - Fix [#408](https://github.com/LayerManager/layman/issues/408) Skip non WMS layers in thumbnail generation. Previously thumbnail generation failed. - Fix [GET Workspace Layer](doc/rest.md#get-workspace-layer) documentation, where was incorrectly `style` item instead of `sld`. +- [#167](https://github.com/LayerManager/layman/issues/167) New metadata property [`spatial_resolution`](doc/metadata.md#spatial_resolution) is available. It has one of two subproperties: + - `scale_denominator` used for vector data + - `ground_sample_distance` used for raster data +- Metadata property `scale_denominator` was removed. Its value is now accessible as subproperty of new [`spatial_resolution`](doc/metadata.md#spatial_resolution) metadata property. ## v1.13.2 2021-06-25 @@ -370,7 +374,7 @@ There is a critical bug in this release, posting new layer breaks Layman: https: - Guess metadata properties - [`md_language`](doc/metadata.md#md_language) of both Layer and Map using pycld2 library - [`language`](doc/metadata.md#language) of Layer using pycld2 library - - [`scale_denominator`](doc/metadata.md#scale_denominator) of Layer using distanced between vertices + - [`scale_denominator`](doc/metadata.md#spatial_resolution) of Layer using distanced between vertices - Change multiplicity of [`language`](doc/metadata.md#language) metadata property from `1` to `1..n` according to XML Schema - Remove [`language`](doc/metadata.md#language) metadata property from Map according to XML Schema - Build Layman as a part of `make start-demo*` commands. diff --git a/Makefile b/Makefile index d00981f75..4a35453df 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ bash: docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm layman_dev bash refresh-doc-metadata-xpath: - docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm layman_dev bash -c "cd src && python3 refresh-doc-metadata-xpath.py" + docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm layman_dev bash -c "cd src && python3 refresh_doc_metadata_xpath.py" bash-root: docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm -u root layman_dev bash diff --git a/doc/metadata.md b/doc/metadata.md index b3ade095e..88e1be25b 100644 --- a/doc/metadata.md +++ b/doc/metadata.md @@ -239,18 +239,37 @@ XPath for Layer: `/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentificat XPath for Map: `/gmd:MD_Metadata/gmd:identificationInfo/srv:SV_ServiceIdentification/gmd:citation/gmd:CI_Citation/gmd:date[gmd:CI_Date/gmd:dateType/gmd:CI_DateTypeCode[@codeList="http://standards.iso.org/iso/19139/resources/gmxCodelists.xml#CI_DateTypeCode" and @codeListValue="revision"]]/gmd:CI_Date/gmd:date/gco:Date/text()` -### scale_denominator -Guessed from distances between vertices of line and polygon features. - +### spatial_resolution Multiplicity: 1 -Shape: Integer +Shape: Object with one of following keys: +- *scale_denominator*: Integer. Scale denominator, used for vector data, guessed from distances between vertices of line and polygon features. +- *ground_sample_distance*: Object. Ground sample distance, used for raster data, read from normalized raster. + Keys: + - **value**: Float. Value of ground sample distance. + - **uom**: String. Unit of measurement of ground sample distance. -Example: `25000` +Example: +```json5 +// Spatial resolution of vector data: +{ + "scale_denominator": 10000 +} +``` + +```json5 +// Spatial resolution of raster data: +{ + "ground_sample_distance": { + "value": 123.45, + "uom": "m" + } +} +``` Synchronizable: yes -XPath for Layer: `/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:spatialResolution/gmd:MD_Resolution/gmd:equivalentScale/gmd:MD_RepresentativeFraction/gmd:denominator/gco:Integer/text()` +XPath for Layer: `/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:spatialResolution` ### title diff --git a/src/layman/__init__.py b/src/layman/__init__.py index a2d9b8a19..6ef262662 100644 --- a/src/layman/__init__.py +++ b/src/layman/__init__.py @@ -114,6 +114,9 @@ pipe.set(LAYMAN_DEPS_ADJUSTED_KEY, 'done') pipe.execute() + elif IN_UTIL_PROCESS: + wait_for_other_process = False # pylint: disable=invalid-name + else: wait_for_other_process = True # pylint: disable=invalid-name except WatchError: diff --git a/src/layman/common/metadata.py b/src/layman/common/metadata.py index b0abcbb4b..f7db2c758 100644 --- a/src/layman/common/metadata.py +++ b/src/layman/common/metadata.py @@ -54,7 +54,7 @@ def extent_equals(ext1, ext2): 'graphic_url': { 'upper_mp': '1', }, - 'scale_denominator': { + 'spatial_resolution': { 'upper_mp': '1', }, 'language': { diff --git a/src/layman/common/micka/util.py b/src/layman/common/micka/util.py index 433e96a00..2f036aca3 100644 --- a/src/layman/common/micka/util.py +++ b/src/layman/common/micka/util.py @@ -595,6 +595,89 @@ def adjust_operates_on(prop_el, prop_value): _add_unknown_reason(prop_el) +def extract_spatial_resolution(prop_els): + result = {} + if prop_els: + prop_el = prop_els[0] + + # scale_denominator + denominator_els = prop_el.xpath('./gmd:MD_Resolution/gmd:equivalentScale/gmd:MD_RepresentativeFraction/' + 'gmd:denominator', namespaces=NAMESPACES) + if denominator_els: + denominator_el = denominator_els[0] + scale_strings = denominator_el.xpath('./gco:Integer/text()', namespaces=NAMESPACES) + scale_denominator = int(scale_strings[0]) if scale_strings else None + result['scale_denominator'] = scale_denominator + + # ground_sample_distance + distance_prop_els = prop_el.xpath('./gmd:MD_Resolution/gmd:distance', namespaces=NAMESPACES) + if distance_prop_els: + ground_sample_distance = None + distance_prop_el = distance_prop_els[0] + distance_value_els = distance_prop_el.xpath('./gco:Distance', namespaces=NAMESPACES) + if distance_value_els: + distance_value_el = distance_value_els[0] + distance_value_strings = distance_value_el.xpath('./text()', namespaces=NAMESPACES) + uom_strings = distance_value_el.xpath('./@uom', namespaces=NAMESPACES) + if distance_value_strings and uom_strings: + distance_value = float(distance_value_strings[0]) + uom = str(uom_strings[0]) + ground_sample_distance = { + 'value': distance_value, + 'uom': uom, + } + result['ground_sample_distance'] = ground_sample_distance + + result = result or None + return result + + +def adjust_spatial_resolution(prop_el, prop_value): + _clear_el(prop_el) + child_el = None + if prop_value is not None: + parser = ET.XMLParser(remove_blank_text=True) + assert set(prop_value.keys()).issubset({'scale_denominator', 'ground_sample_distance'}) and len(prop_value) <= 1 + if 'scale_denominator' in prop_value: + scale_denominator = prop_value['scale_denominator'] + denominator_el_str = f""" + + {str(scale_denominator)} + + """ if scale_denominator is not None else '' + child_el = ET.fromstring(f""" + + + + {denominator_el_str} + + + + """, parser=parser) + if 'ground_sample_distance' in prop_value: + ground_sample_distance = prop_value['ground_sample_distance'] + if ground_sample_distance is not None: + distance_value = ground_sample_distance.get('value') + uom = ground_sample_distance.get('uom') + assert distance_value is not None and uom is not None + distance_el_str = f""" + + {escape(str(distance_value))} + + """ + else: + distance_el_str = '' + child_el = ET.fromstring(f""" + + {distance_el_str} + + """, parser=parser) + if child_el: + prop_el.append(child_el) + else: + _add_unknown_reason(prop_el) + + def get_record_element_by_id(csw, ident): csw.getrecordbyid(id=[ident], esn='full', outputschema=NAMESPACES['gmd']) xml = csw._exml # pylint: disable=protected-access diff --git a/src/layman/layer/__init__.py b/src/layman/layer/__init__.py index 421200e2d..5acc8bf50 100644 --- a/src/layman/layer/__init__.py +++ b/src/layman/layer/__init__.py @@ -86,7 +86,7 @@ def get_layer_info_keys(file_type): 'language', 'revision_date', 'reference_system', - 'scale_denominator', + 'spatial_resolution', 'title', 'wfs_url', 'wms_url', diff --git a/src/layman/layer/filesystem/gdal.py b/src/layman/layer/filesystem/gdal.py index ed3671b83..d0a03e5e8 100644 --- a/src/layman/layer/filesystem/gdal.py +++ b/src/layman/layer/filesystem/gdal.py @@ -289,3 +289,11 @@ def get_bbox(workspace, layer): miny = maxy + geo_transform[5] * data.RasterYSize result = (minx, miny, maxx, maxy) return result + + +def get_normalized_ground_sample_distance(workspace, layer): + filepath = get_normalized_raster_layer_main_filepath(workspace, layer) + pixel_size = get_pixel_size(filepath) + abs_pixel_size = [abs(size) for size in pixel_size] + distance_value = sum(abs_pixel_size) / len(abs_pixel_size) + return distance_value diff --git a/src/layman/layer/micka/csw.py b/src/layman/layer/micka/csw.py index 815964b18..62f9bda7e 100644 --- a/src/layman/layer/micka/csw.py +++ b/src/layman/layer/micka/csw.py @@ -9,6 +9,7 @@ from layman.common.filesystem.uuid import get_publication_uuid_file from layman.common.micka import util as common_util from layman.common import language as common_language, empty_method, empty_method_returns_none, bbox as bbox_util +from layman.layer.filesystem import gdal from layman.layer.filesystem.uuid import get_layer_uuid from layman.layer import db from layman.layer.geoserver import wms @@ -135,7 +136,8 @@ def get_template_path_and_values(workspace, layername, http_method=None): abstract or '' ]))), None) - if publ_info.get('file', dict()).get('file_type') == settings.FILE_TYPE_VECTOR: + file_type = publ_info.get('file', dict()).get('file_type') + if file_type == settings.FILE_TYPE_VECTOR: try: languages = db.get_text_languages(workspace, layername) except LaymanError: @@ -144,9 +146,20 @@ def get_template_path_and_values(workspace, layername, http_method=None): scale_denominator = db.guess_scale_denominator(workspace, layername) except LaymanError: scale_denominator = None - else: + spatial_resolution = { + 'scale_denominator': scale_denominator, + } + elif file_type == settings.FILE_TYPE_RASTER: languages = [] - scale_denominator = None + distance_value = gdal.get_normalized_ground_sample_distance(workspace, layername) + spatial_resolution = { + 'ground_sample_distance': { + 'value': distance_value, + 'uom': 'm', # EPSG:3857 + } + } + else: + raise NotImplementedError(f"Unknown file type: {file_type}") prop_values = _get_property_values( workspace=workspace, @@ -166,7 +179,7 @@ def get_template_path_and_values(workspace, layername, http_method=None): organisation_name=None, md_language=md_language, languages=languages, - scale_denominator=scale_denominator, + spatial_resolution=spatial_resolution, epsg_codes=settings.LAYMAN_OUTPUT_SRS_LIST, ) if http_method == common.REQUEST_METHOD_POST: @@ -192,7 +205,7 @@ def _get_property_values( wms_url="http://www.env.cz/corine/data/download.zip", wfs_url="http://www.env.cz/corine/data/download.zip", epsg_codes=None, - scale_denominator=None, + spatial_resolution=None, languages=None, md_language=None, ): @@ -220,7 +233,7 @@ def _get_property_values( 'wms_url': f"{wms.add_capabilities_params_to_url(wms_url)}&LAYERS={layername}", 'wfs_url': f"{wfs.add_capabilities_params_to_url(wfs_url)}&LAYERS={layername}", 'layer_endpoint': url_for('rest_workspace_layer.get', workspace=workspace, layername=layername), - 'scale_denominator': scale_denominator, + 'spatial_resolution': spatial_resolution, 'language': languages, 'md_organisation_name': md_organisation_name, 'organisation_name': organisation_name, @@ -317,12 +330,12 @@ def _get_property_values( 'xpath_extract_fn': lambda l: l[0] if l else None, 'adjust_property_element': common_util.adjust_graphic_url, }, - 'scale_denominator': { - 'xpath_parent': '/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:spatialResolution/gmd:MD_Resolution/gmd:equivalentScale/gmd:MD_RepresentativeFraction', - 'xpath_property': './gmd:denominator', - 'xpath_extract': './gco:Integer/text()', - 'xpath_extract_fn': lambda l: int(l[0]) if l else None, - 'adjust_property_element': common_util.adjust_integer, + 'spatial_resolution': { + 'xpath_parent': '/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification', + 'xpath_property': './gmd:spatialResolution', + 'xpath_extract': '.', + 'xpath_extract_fn': common_util.extract_spatial_resolution, + 'adjust_property_element': common_util.adjust_spatial_resolution, }, 'language': { 'xpath_parent': '/gmd:MD_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification', @@ -390,7 +403,7 @@ def get_metadata_comparison(workspace, layername): 'publication_date', 'revision_date', 'reference_system', - 'scale_denominator', + 'spatial_resolution', 'title', 'wfs_url', 'wms_url', diff --git a/src/layman/layer/micka/record-template-filled.xml b/src/layman/layer/micka/record-template-filled.xml index 10a1177d8..c81279123 100644 --- a/src/layman/layer/micka/record-template-filled.xml +++ b/src/layman/layer/micka/record-template-filled.xml @@ -172,11 +172,9 @@ - - - - - + + 123.45 + diff --git a/src/layman/layer/micka/record-template.xml b/src/layman/layer/micka/record-template.xml index 9abd14c20..c5b1b5bfd 100644 --- a/src/layman/layer/micka/record-template.xml +++ b/src/layman/layer/micka/record-template.xml @@ -134,15 +134,7 @@ xsi:schemaLocation="http://www.isotc211.org/2005/gmd http://schemas.opengis.net/ vector - - - - - - - - - + diff --git a/src/layman/layer/micka/util_test.py b/src/layman/layer/micka/util_test.py index 41a94ca6a..b7edb4af3 100644 --- a/src/layman/layer/micka/util_test.py +++ b/src/layman/layer/micka/util_test.py @@ -40,7 +40,11 @@ def test_fill_template(): except OSError: pass file_object = common_util.fill_xml_template_as_pretty_file_object('src/layman/layer/micka/record-template.xml', - _get_property_values(), METADATA_PROPERTIES) + _get_property_values( + spatial_resolution={ + 'scale_denominator': None, + } + ), METADATA_PROPERTIES) with open(xml_path, 'wb') as out: out.write(file_object.read()) @@ -68,7 +72,7 @@ def test_parse_md_properties(): 'organisation_name', 'publication_date', 'reference_system', - 'scale_denominator', + 'spatial_resolution', 'title', 'wfs_url', 'wms_url', @@ -79,7 +83,9 @@ def test_parse_md_properties(): 'md_date_stamp': '2007-05-25', 'md_organisation_name': None, 'organisation_name': None, - 'scale_denominator': None, + 'spatial_resolution': { + 'scale_denominator': None, + }, 'language': [], 'reference_system': [4326, 3857], 'title': 'CORINE - Krajinný pokryv CLC 90', @@ -122,7 +128,12 @@ def test_fill_xml_template(): 'abstract': None, 'organisation_name': 'My Organization', 'graphic_url': 'https://example.com/myimage.png', - 'scale_denominator': None, + 'spatial_resolution': { + 'ground_sample_distance': { + 'value': 123.45, + 'uom': "m", + } + }, 'language': ['cze', 'eng'], 'extent': [11.87, 48.12, 19.13, 51.59], 'wms_url': 'https://example.com/wms', diff --git a/src/layman/layer/rest_workspace_test.py b/src/layman/layer/rest_workspace_test.py index 7a0a32519..f3ba60288 100644 --- a/src/layman/layer/rest_workspace_test.py +++ b/src/layman/layer/rest_workspace_test.py @@ -45,7 +45,7 @@ 'publication_date', 'reference_system', 'revision_date', - 'scale_denominator', + 'spatial_resolution', 'title', 'wfs_url', 'wms_url', @@ -336,7 +336,9 @@ def test_post_layers_simple(client): 'publication_date': TODAY_DATE, 'reference_system': [3857, 4326, 5514], 'revision_date': None, - 'scale_denominator': 100000000, + 'spatial_resolution': { + 'scale_denominator': 100000000, + }, 'title': 'ne_110m_admin_0_countries', } check_metadata(client, username, layername, METADATA_PROPERTIES_EQUAL, expected_md_values) @@ -587,7 +589,9 @@ def test_post_layers_complex(client): 'publication_date': TODAY_DATE, 'reference_system': [3857, 4326, 5514], 'revision_date': None, - 'scale_denominator': 100000000, + 'spatial_resolution': { + 'scale_denominator': 100000000, + }, 'title': "staty", } check_metadata(client, username, layername, METADATA_PROPERTIES_EQUAL, expected_md_values) @@ -742,7 +746,9 @@ def test_patch_layer_title(client): 'publication_date': TODAY_DATE, 'reference_system': [3857, 4326, 5514], 'revision_date': TODAY_DATE, - 'scale_denominator': 100000000, + 'spatial_resolution': { + 'scale_denominator': 100000000, + }, 'title': "New Title of Countries", } check_metadata(client, username, layername, METADATA_PROPERTIES_EQUAL, expected_md_values) @@ -807,7 +813,9 @@ def test_patch_layer_style(client): 'publication_date': TODAY_DATE, 'reference_system': [3857, 4326, 5514], 'revision_date': TODAY_DATE, - 'scale_denominator': 100000000, + 'spatial_resolution': { + 'scale_denominator': 100000000, + }, 'title': 'countries in blue', } check_metadata(client, username, layername, METADATA_PROPERTIES_EQUAL, expected_md_values) @@ -952,7 +960,7 @@ def test_patch_layer_data(client): 'publication_date': TODAY_DATE, 'reference_system': [3857, 4326, 5514], 'revision_date': TODAY_DATE, - 'scale_denominator': None, + 'spatial_resolution': None, # it's point data now and we can't guess scale from point data 'title': 'populated places', } check_metadata(client, username, layername, METADATA_PROPERTIES_EQUAL, expected_md_values) diff --git a/test_tools/data/micka/rest_test_filled_raster_template.xml b/test_tools/data/micka/rest_test_filled_raster_template.xml new file mode 100644 index 000000000..bbf32e956 --- /dev/null +++ b/test_tools/data/micka/rest_test_filled_raster_template.xml @@ -0,0 +1,277 @@ + + + + m-81c0debe-b2ea-4829-9b16-581083b29907 + + + eng + + + dataset + + + + + + + + + + + + + + + + + + + + pointOfContact + + + + + 2007-05-25 + + + ISO 19115/INSPIRE_TG2/CZ4 + + + 2003/cor.1/2006 + + + + + + + EPSG:4326 + + + + + + + + + + + EPSG:3857 + + + + + + + + + + + EPSG:5514 + + + + + + + + + + + post_tif_rgba_4326 + + + + + 2019-12-07 + + + publication + + + + + + + post_tif_rgba_4326 + + + + + + + + + + + + + + + + + + + + + + + + + custodian + + + + + + + http://enjoychallenge.tech/rest/workspaces/test_workspace/layers/post_tif_rgba_4326/thumbnail + + + PNG + + + + + + + + + + GEMET - INSPIRE themes, version 1.0 + + + + + 2008-06-01 + + + publication + + + + + + + + + + + otherRestrictions + + + Bez omezení + + + + + + + otherRestrictions + + + Žádné podmínky neplatí + + + + + vector + + + + + 0.043745341126665845 + + + + + + utf8 + + + + + + + + 15.08622670719884 + + + 15.086453058544048 + + + 50.665636410141985 + + + 50.66575821142854 + + + + + + + + + + + + + + + http://localhost:8000/geoserver/test_workspace_wms/ows?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&LAYERS=post_tif_rgba_4326 + + + OGC:WMS-1.3.0-http-get-capabilities + + + download + + + + + + + http://localhost:8000/geoserver/test_workspace/wfs?SERVICE=WFS&REQUEST=GetCapabilities&VERSION=2.0.0&LAYERS=post_tif_rgba_4326 + + + OGC:WFS-2.0.0-http-get-capabilities + + + download + + + + + + + http://enjoychallenge.tech/rest/workspaces/test_workspace/layers/post_tif_rgba_4326 + + + WWW:LINK-1.0-http--link + + + information + + + + + + + + + + + + + dataset + + + + + + + + + + + diff --git a/tests/static_publications/__init__.py b/tests/static_publications/__init__.py index dc2375edf..d2eb7bd76 100644 --- a/tests/static_publications/__init__.py +++ b/tests/static_publications/__init__.py @@ -17,6 +17,21 @@ USERS = {OWNER, OWNER2, NOT_OWNER, } HEADERS = {user: process_client.get_authz_headers(user) for user in USERS} +MICKA_XML_LAYER_DIFF_LINES = [ + {'plus_line': '+ m-81c0debe-b2ea-4829-9b16-581083b29907\n', + 'minus_line_starts_with': '- m', + 'minus_line_ends_with': '\n', + }, + {'plus_line': '+ 2007-05-25\n', + 'minus_line_starts_with': '- ', + 'minus_line_ends_with': '\n', + }, + {'plus_line': '+ 2019-12-07\n', + 'minus_line_starts_with': '- ', + 'minus_line_ends_with': '\n', + }, +] + PUBLICATIONS = { ################################################################################ # LAYERS @@ -259,6 +274,10 @@ 'thumbnail': '/code/test_tools/data/thumbnail/raster_layer_tif_rgba_4326.png', 'file_type': settings.FILE_TYPE_RASTER, 'style_type': 'sld', + 'micka_xml': {'filled_template': 'test_tools/data/micka/rest_test_filled_raster_template.xml', + 'diff_lines': MICKA_XML_LAYER_DIFF_LINES, + 'diff_lines_len': 29, + }, }, }, (COMMON_WORKSPACE, LAYER_TYPE, 'post_tiff'): { @@ -428,20 +447,7 @@ 'file_type': settings.FILE_TYPE_VECTOR, 'style_type': 'sld', 'micka_xml': {'filled_template': 'test_tools/data/micka/rest_test_filled_template.xml', - 'diff_lines': [ - {'plus_line': '+ m-81c0debe-b2ea-4829-9b16-581083b29907\n', - 'minus_line_starts_with': '- m', - 'minus_line_ends_with': '\n', - }, - {'plus_line': '+ 2007-05-25\n', - 'minus_line_starts_with': '- ', - 'minus_line_ends_with': '\n', - }, - {'plus_line': '+ 2019-12-07\n', - 'minus_line_starts_with': '- ', - 'minus_line_ends_with': '\n', - }, - ], + 'diff_lines': MICKA_XML_LAYER_DIFF_LINES, 'diff_lines_len': 29, }, },