From bf4009eebdf4a8d0ddb2020922367dc2c97d16ac Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 21 Jun 2021 14:53:16 +0200 Subject: [PATCH 1/7] Do not store raster layers in DB --- doc/rest.md | 17 +++++++++-------- src/layman/layer/db/tasks.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/doc/rest.md b/doc/rest.md index ad07b4d10..39dd31857 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -84,14 +84,15 @@ Publish vector data file as new layer of WMS and WFS. Processing chain consists of few steps: - save file to workspace directory within Layman data directory - save basic information (name, title, access_rights) into PostgreSQL -- import the file to PostgreSQL database as new table into workspace schema, including geometry transformation to EPSG:3857 -- publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer -- save bounding box into PostgreSQL -- for layers with SLD or none style: - - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer -- else for layers with QML style: - - create QGS file on QGIS server filesystem with appropriate style - - publish the layer on GeoServer through WMS cascade from QGIS server +- for vector layers import the file to PostgreSQL database as new table into workspace schema, including geometry transformation to EPSG:3857 +- for vector layers publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer +- for vector layers save bounding box into PostgreSQL +- for vector layers + - for layers with SLD or none style: + - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer + - else for layers with QML style: + - create QGS file on QGIS server filesystem with appropriate style + - publish the layer on GeoServer through WMS cascade from QGIS server - generate thumbnail image - publish metadata record to Micka (it's public if and only if read access is set to EVERYONE) - update thumbnail of each [map](models.md#map) that points to this layer diff --git a/src/layman/layer/db/tasks.py b/src/layman/layer/db/tasks.py index c05738bf3..924dfe822 100644 --- a/src/layman/layer/db/tasks.py +++ b/src/layman/layer/db/tasks.py @@ -3,9 +3,9 @@ from layman.celery import AbortedException from layman.common import empty_method_returns_true from layman.layer.filesystem.input_file import get_layer_main_file_path -from layman import celery_app +from layman import celery_app, util as layman_util, settings from layman.http import LaymanError -from .. import db +from .. import db, LAYER_TYPE from .table import delete_layer @@ -30,6 +30,14 @@ def refresh_table( db.ensure_workspace(username) if self.is_aborted(): raise AbortedException + + file_type = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']})['file']['file_type'] + if file_type != settings.FILE_TYPE_VECTOR: + return + + if self.is_aborted(): + raise AbortedException + main_filepath = get_layer_main_file_path(username, layername) process = db.import_layer_vector_file_async(username, layername, main_filepath, crs_id) while process.poll() is None and not self.is_aborted(): From 7da8189fd75ec82eaed8449c7166070bd41e8f74 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 21 Jun 2021 15:02:31 +0200 Subject: [PATCH 2/7] Do not calculate bbox for raster layers --- doc/rest.md | 13 ++++++------- src/layman/layer/prime_db_schema/bbox_tasks.py | 4 +++- src/layman/layer/prime_db_schema/tasks.py | 8 ++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/doc/rest.md b/doc/rest.md index 39dd31857..398bf927d 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -85,14 +85,13 @@ Processing chain consists of few steps: - save file to workspace directory within Layman data directory - save basic information (name, title, access_rights) into PostgreSQL - for vector layers import the file to PostgreSQL database as new table into workspace schema, including geometry transformation to EPSG:3857 -- for vector layers publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer +- publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer - for vector layers save bounding box into PostgreSQL -- for vector layers - - for layers with SLD or none style: - - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer - - else for layers with QML style: - - create QGS file on QGIS server filesystem with appropriate style - - publish the layer on GeoServer through WMS cascade from QGIS server +- for layers with SLD or none style: + - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer +- else for layers with QML style: + - create QGS file on QGIS server filesystem with appropriate style + - publish the layer on GeoServer through WMS cascade from QGIS server - generate thumbnail image - publish metadata record to Micka (it's public if and only if read access is set to EVERYONE) - update thumbnail of each [map](models.md#map) that points to this layer diff --git a/src/layman/layer/prime_db_schema/bbox_tasks.py b/src/layman/layer/prime_db_schema/bbox_tasks.py index 6d000e733..871c4ea39 100644 --- a/src/layman/layer/prime_db_schema/bbox_tasks.py +++ b/src/layman/layer/prime_db_schema/bbox_tasks.py @@ -1,7 +1,7 @@ from celery.utils.log import get_task_logger from layman.celery import AbortedException -from layman import celery_app +from layman import celery_app, util as layman_util, settings from .. import LAYER_TYPE from ..db import get_bbox as db_get_bbox from ...common.prime_db_schema.publications import set_bbox @@ -22,6 +22,8 @@ def patch_after_feature_change( if self.is_aborted(): raise AbortedException + file_type = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']})['file']['file_type'] + assert file_type == settings.FILE_TYPE_VECTOR bbox = db_get_bbox(username, layername) if self.is_aborted(): diff --git a/src/layman/layer/prime_db_schema/tasks.py b/src/layman/layer/prime_db_schema/tasks.py index e2a95c305..7ac4334a9 100644 --- a/src/layman/layer/prime_db_schema/tasks.py +++ b/src/layman/layer/prime_db_schema/tasks.py @@ -2,7 +2,7 @@ from layman.celery import AbortedException from layman.common import empty_method_returns_true -from layman import celery_app +from layman import celery_app, util as layman_util, settings from .. import LAYER_TYPE from ..db import get_bbox as db_get_bbox from ...common.prime_db_schema.publications import set_bbox @@ -25,7 +25,11 @@ def refresh_bbox( if self.is_aborted(): raise AbortedException - bbox = db_get_bbox(username, layername) + file_type = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']})['file']['file_type'] + if file_type == settings.FILE_TYPE_VECTOR: + bbox = db_get_bbox(username, layername) + else: + bbox = None if self.is_aborted(): raise AbortedException From 343ca7f0ecc3d84b07fc3fb005b99dbc0eb4e963 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 21 Jun 2021 15:03:05 +0200 Subject: [PATCH 3/7] Do not store raster layers with WFS source --- doc/rest.md | 2 +- src/layman/layer/geoserver/tasks.py | 4 ++++ src/layman/layer/geoserver/wfs_tasks.py | 8 ++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/rest.md b/doc/rest.md index 398bf927d..2e01bc5b5 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -85,7 +85,7 @@ Processing chain consists of few steps: - save file to workspace directory within Layman data directory - save basic information (name, title, access_rights) into PostgreSQL - for vector layers import the file to PostgreSQL database as new table into workspace schema, including geometry transformation to EPSG:3857 -- publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer +- for vector layers publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer - for vector layers save bounding box into PostgreSQL - for layers with SLD or none style: - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer diff --git a/src/layman/layer/geoserver/tasks.py b/src/layman/layer/geoserver/tasks.py index fa515170c..8507a9ea0 100644 --- a/src/layman/layer/geoserver/tasks.py +++ b/src/layman/layer/geoserver/tasks.py @@ -80,6 +80,10 @@ def refresh_wfs( ensure_user=False, access_rights=None, ): + file_type = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']})['file']['file_type'] + if file_type != settings.FILE_TYPE_VECTOR: + return + if description is None: description = layername if title is None: diff --git a/src/layman/layer/geoserver/wfs_tasks.py b/src/layman/layer/geoserver/wfs_tasks.py index 3bd406e0b..c96688263 100644 --- a/src/layman/layer/geoserver/wfs_tasks.py +++ b/src/layman/layer/geoserver/wfs_tasks.py @@ -1,8 +1,8 @@ from geoserver import util as gs_util -from layman import celery_app, settings +from layman import celery_app, settings, util as layman_util from layman.celery import AbortedException from . import wfs -from .. import geoserver +from .. import geoserver, LAYER_TYPE headers_json = geoserver.headers_json @@ -20,6 +20,10 @@ def patch_after_feature_change( if self.is_aborted(): raise AbortedException + file_type = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['file']})['file']['file_type'] + if file_type != settings.FILE_TYPE_VECTOR: + return + bbox = geoserver.get_layer_bbox(workspace, layer) gs_util.patch_feature_type(workspace, layer, auth=settings.LAYMAN_GS_AUTH, bbox=bbox) wfs.clear_cache(workspace) From 672ee94a4fa9f5936ee34c94994d0026f5a33003 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 21 Jun 2021 15:03:12 +0200 Subject: [PATCH 4/7] Do not store raster layers with WMS source --- doc/rest.md | 11 +++-- src/layman/layer/geoserver/tasks.py | 64 +++++++++++++------------ src/layman/layer/geoserver/wms_tasks.py | 16 ++++--- 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/doc/rest.md b/doc/rest.md index 2e01bc5b5..39dd31857 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -87,11 +87,12 @@ Processing chain consists of few steps: - for vector layers import the file to PostgreSQL database as new table into workspace schema, including geometry transformation to EPSG:3857 - for vector layers publish the table as new layer (feature type) within appropriate WFS workspaces of GeoServer - for vector layers save bounding box into PostgreSQL -- for layers with SLD or none style: - - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer -- else for layers with QML style: - - create QGS file on QGIS server filesystem with appropriate style - - publish the layer on GeoServer through WMS cascade from QGIS server +- for vector layers + - for layers with SLD or none style: + - publish the table as new layer (feature type) within appropriate WMS workspaces of GeoServer + - else for layers with QML style: + - create QGS file on QGIS server filesystem with appropriate style + - publish the layer on GeoServer through WMS cascade from QGIS server - generate thumbnail image - publish metadata record to Micka (it's public if and only if read access is set to EVERYONE) - update thumbnail of each [map](models.md#map) that points to this layer diff --git a/src/layman/layer/geoserver/tasks.py b/src/layman/layer/geoserver/tasks.py index 8507a9ea0..c90aea22a 100644 --- a/src/layman/layer/geoserver/tasks.py +++ b/src/layman/layer/geoserver/tasks.py @@ -2,10 +2,10 @@ from geoserver import util as gs_util from layman.celery import AbortedException -from layman import celery_app, settings +from layman import celery_app, settings, util as layman_util from layman.common import empty_method_returns_true from . import wms, wfs, sld -from .. import geoserver +from .. import geoserver, LAYER_TYPE logger = get_task_logger(__name__) @@ -30,35 +30,37 @@ def refresh_wms( ensure_user=False, access_rights=None, ): - if description is None: - description = layername - if title is None: - title = layername - geoserver_workspace = wms.get_geoserver_workspace(username) - if ensure_user: - geoserver.ensure_workspace(username) - - if self.is_aborted(): - raise AbortedException - if store_in_geoserver: - gs_util.delete_wms_layer(geoserver_workspace, layername, settings.LAYMAN_GS_AUTH) - gs_util.delete_wms_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, wms.get_qgis_store_name(layername)) - geoserver.publish_layer_from_db(username, - layername, - description, - title, - access_rights, - geoserver_workspace=geoserver_workspace, - ) - else: - gs_util.delete_feature_type(geoserver_workspace, layername, settings.LAYMAN_GS_AUTH) - geoserver.publish_layer_from_qgis(username, - layername, - description, - title, - access_rights, - geoserver_workspace=geoserver_workspace, - ) + file_type = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']})['file']['file_type'] + if file_type == settings.FILE_TYPE_VECTOR: + if description is None: + description = layername + if title is None: + title = layername + geoserver_workspace = wms.get_geoserver_workspace(username) + if ensure_user: + geoserver.ensure_workspace(username) + + if self.is_aborted(): + raise AbortedException + if store_in_geoserver: + gs_util.delete_wms_layer(geoserver_workspace, layername, settings.LAYMAN_GS_AUTH) + gs_util.delete_wms_store(geoserver_workspace, settings.LAYMAN_GS_AUTH, wms.get_qgis_store_name(layername)) + geoserver.publish_layer_from_db(username, + layername, + description, + title, + access_rights, + geoserver_workspace=geoserver_workspace, + ) + else: + gs_util.delete_feature_type(geoserver_workspace, layername, settings.LAYMAN_GS_AUTH) + geoserver.publish_layer_from_qgis(username, + layername, + description, + title, + access_rights, + geoserver_workspace=geoserver_workspace, + ) wms.clear_cache(username) if self.is_aborted(): diff --git a/src/layman/layer/geoserver/wms_tasks.py b/src/layman/layer/geoserver/wms_tasks.py index 2cc59ecf7..248f56cea 100644 --- a/src/layman/layer/geoserver/wms_tasks.py +++ b/src/layman/layer/geoserver/wms_tasks.py @@ -20,13 +20,15 @@ def patch_after_feature_change( if self.is_aborted(): raise AbortedException - bbox = geoserver.get_layer_bbox(workspace, layer) - geoserver_workspace = wms.get_geoserver_workspace(workspace) - style_type = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['style_type'], })['style_type'] - if style_type == 'sld': - gs_util.patch_feature_type(geoserver_workspace, layer, auth=settings.LAYMAN_GS_AUTH, bbox=bbox) - elif style_type == 'qml': - gs_util.patch_wms_layer(geoserver_workspace, layer, auth=settings.LAYMAN_GS_AUTH, bbox=bbox) + file_type = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['file']})['file']['file_type'] + if file_type == settings.FILE_TYPE_VECTOR: + bbox = geoserver.get_layer_bbox(workspace, layer) + geoserver_workspace = wms.get_geoserver_workspace(workspace) + style_type = layman_util.get_publication_info(workspace, LAYER_TYPE, layer, context={'keys': ['style_type'], })['style_type'] + if style_type == 'sld': + gs_util.patch_feature_type(geoserver_workspace, layer, auth=settings.LAYMAN_GS_AUTH, bbox=bbox) + elif style_type == 'qml': + gs_util.patch_wms_layer(geoserver_workspace, layer, auth=settings.LAYMAN_GS_AUTH, bbox=bbox) wms.clear_cache(workspace) From 5b5c09f796c82e902570eae71cc0a1cb5de87dc2 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 21 Jun 2021 15:23:00 +0200 Subject: [PATCH 5/7] Changelog for raster layer in vector sources --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae0635c29..763c56d16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ ### Changes - [#167](https://github.com/LayerManager/layman/issues/167) Allow publishing also raster geospatial data from `GeoTiff` and `JPEG2000` formats using [POST Workspace Layers](doc/rest.md#post-workspace-layers) and [PATCH Workspace Layer](doc/rest.md#patch-workspace-layer). - [#167](https://github.com/LayerManager/layman/issues/167) Add `file_type` item to `file` item in [GET Workspace Layer](doc/rest.md#get-workspace-layer) response to distinguish raster and vector layer. +- [#167](https://github.com/LayerManager/layman/issues/167) Raster layers are not stored in DB table, for WFS and for WMS. Also bounding box is not calculated for them. - [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/gdal/docker). - [#367](https://github.com/LayerManager/layman/issues/367) Upgrade also - python from 3.6 to 3.8 From f5f2406aada31f73bf1da93e305ec811487ff6fb Mon Sep 17 00:00:00 2001 From: index-git Date: Tue, 22 Jun 2021 10:09:34 +0200 Subject: [PATCH 6/7] Set timeout to 60 for test on GitHub CI --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 36d065f32..796495c7b 100644 --- a/test.sh +++ b/test.sh @@ -9,7 +9,7 @@ rm -rf tmp/artifacts/* if [ "$CI" == "true" ] then - python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -m "not irritating" -W ignore::DeprecationWarning -xvv + python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -m "not irritating" --timeout=60 -W ignore::DeprecationWarning -xvv else python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xvv fi From ef113541549f7976d483a6212dea26f633c38bbb Mon Sep 17 00:00:00 2001 From: index-git Date: Tue, 22 Jun 2021 10:35:31 +0200 Subject: [PATCH 7/7] Mark src/layman/layer/micka/util_test.py::test_num_records as irritating --- src/layman/layer/micka/util_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/layman/layer/micka/util_test.py b/src/layman/layer/micka/util_test.py index 02aa297e6..5ca428b35 100644 --- a/src/layman/layer/micka/util_test.py +++ b/src/layman/layer/micka/util_test.py @@ -140,6 +140,7 @@ def test_fill_xml_template(): @pytest.mark.usefixtures('app_context', 'ensure_layman', 'client') +@pytest.mark.irritating def test_num_records(): publs_by_type = uuid.check_redis_consistency() num_publications = sum([len(publs) for publs in publs_by_type.values()])