diff --git a/CHANGELOG.md b/CHANGELOG.md index 68502ca83..6002ce38f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,7 +103,7 @@ make timgen-build - [#383](https://github.com/LayerManager/layman/issues/383) Add new Makefile target `upgrade-after-timeout` to finish upgrade in case of GeoServer call timeout. - Fix [GET Workspace Layer](doc/rest.md#get-workspace-layer) documentation; `style` item was incorrectly used instead of `sld`. - [#347](https://github.com/LayerManager/layman/issues/347) Upgrade PostgreSQL 10 to 13.3 and PostGIS 2.4 to 3.1. Use docker image from [layermanager/postgis@hub.docker.com](https://hub.docker.com/repository/docker/layermanager/postgis@github.com), source is located at [layermanager/docker-postgis@github.com](https://github.com/LayerManager/docker-postgis). -- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/gdal/docker). +- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/docker). - [#367](https://github.com/LayerManager/layman/issues/367) Upgrade also - python from 3.6 to 3.8 - flask from 1.1 to 2.0 diff --git a/doc/dependencies.md b/doc/dependencies.md index 1687c2a8c..fc63e0ab3 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -78,7 +78,7 @@ | [postgis_restore.pl](https://github.com/postgis/postgis/blob/3.1.2/utils/postgis_restore.pl.in) | GNU GPL v2 | upgrade_v1_14_postgis_restore.pl | upgrade | src | | | [libxml2](http://xmlsoft.org/) | MIT | python3-lxml | prod | bin | | | [libxslt1.1](http://xmlsoft.org/libxslt/) | MIT | python3-lxml | prod | bin | | -| [gdal/docker](https://github.com/OSGeo/gdal/tree/master/gdal/docker) | MIT License | Dockerfile | prod | bin | | +| [gdal/docker](https://github.com/OSGeo/gdal/tree/master/docker) | MIT License | Dockerfile | prod | bin | | | [kartoza/docker-geoserver](https://github.com/kartoza/docker-geoserver) | GNU GPL v2 | docker-compose.yml | dev | bin | | | [jirikcz/qgis-server](https://github.com/LayerManager/docker-qgis-server) | GNU GPL v3 | docker-compose.yml | dev | bin | | | [layermanager/docker-postgis](https://github.com/layermanager/docker-postgis) | MIT | docker-compose.yml | dev | bin | | diff --git "a/sample/layman.layer/ne_110m_admin_0_boundary lines land +\304\233\305\241\304\215\305\231\305\276\303\275\303\241\303\255.zip" "b/sample/layman.layer/ne_110m_admin_0_boundary lines land +\304\233\305\241\304\215\305\231\305\276\303\275\303\241\303\255.zip" deleted file mode 100644 index 2f11dc313..000000000 Binary files "a/sample/layman.layer/ne_110m_admin_0_boundary lines land +\304\233\305\241\304\215\305\231\305\276\303\275\303\241\303\255.zip" and /dev/null differ diff --git a/sample/layman.layer/sample_tif_colortable_nodata_opaque.zip b/sample/layman.layer/sample_tif_colortable_nodata_opaque.zip deleted file mode 100644 index 0bb8cc767..000000000 Binary files a/sample/layman.layer/sample_tif_colortable_nodata_opaque.zip and /dev/null differ diff --git a/sample/layman.layer/sample_tif_tfw_rgba_opaque.zip b/sample/layman.layer/sample_tif_tfw_rgba_opaque.zip deleted file mode 100644 index 77bd8cfce..000000000 Binary files a/sample/layman.layer/sample_tif_tfw_rgba_opaque.zip and /dev/null differ diff --git a/sample/layman.layer/small_layer.zip b/sample/layman.layer/small_layer.zip deleted file mode 100644 index 92a7eb843..000000000 Binary files a/sample/layman.layer/small_layer.zip and /dev/null differ diff --git a/test_tools/process_client.py b/test_tools/process_client.py index fc8ccac12..93138c680 100644 --- a/test_tools/process_client.py +++ b/test_tools/process_client.py @@ -6,12 +6,15 @@ from functools import partial from collections import namedtuple import xml.etree.ElementTree as ET +import tempfile +import shutil import requests from geoserver import error as gs_error from layman import app, settings, util as layman_util from layman.layer.geoserver import wfs, wms from layman.http import LaymanError +from . import util from .util import url_for from .process import LAYMAN_CELERY_QUEUE @@ -75,6 +78,13 @@ ), } +# pylint: disable=unexpected-keyword-arg +CompressTypeDef = namedtuple('CompressTypeDef', [ + 'archive_name', + 'inner_directory', + 'file_name', +], defaults=[None, None, None]) + def wait_for_rest(url, max_attempts, sleeping_time, check_response, headers=None): headers = headers or None @@ -120,6 +130,7 @@ def upload_file_chunks(publication_type, ) file_chunks = [('file', file_name) for file_name in file_paths] + file_dict = None for file_type, file_name in file_chunks: try: basename = os.path.basename(file_name) @@ -137,7 +148,8 @@ def upload_file_chunks(publication_type, data=data) raise_layman_error(chunk_response) finally: - file_dict[file_type][1].close() + if file_dict: + file_dict[file_type][1].close() def patch_workspace_publication(publication_type, @@ -150,6 +162,8 @@ def patch_workspace_publication(publication_type, title=None, style_file=None, check_response_fn=None, + compress=False, + compress_settings=None, with_chunks=False, ): headers = headers or {} @@ -160,12 +174,20 @@ def patch_workspace_publication(publication_type, # Only Layer files can be uploaded by chunks assert not with_chunks or publication_type == LAYER_TYPE + # Compress settings can be used only with compress option + assert not compress_settings or compress with app.app_context(): r_url = url_for(publication_type_def.patch_workspace_publication_url, workspace=workspace, **{publication_type_def.url_param_name: name}) + temp_dir = None + if compress: + temp_dir = tempfile.mkdtemp(prefix="layman_zip_") + zip_file = util.compress_files(file_paths, compress_settings=compress_settings, output_dir=temp_dir) + file_paths = [zip_file] + for file_path in file_paths: assert os.path.isfile(file_path), file_path files = [] @@ -204,6 +226,8 @@ def patch_workspace_publication(publication_type, wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers) wfs.clear_cache(workspace) wms.clear_cache(workspace) + if temp_dir: + shutil.rmtree(temp_dir) return response.json() @@ -254,6 +278,8 @@ def publish_workspace_publication(publication_type, description=None, check_response_fn=None, with_chunks=False, + compress=False, + compress_settings=None, crs=None, ): title = title or name @@ -265,10 +291,18 @@ def publish_workspace_publication(publication_type, # Only Layer files can be uploaded by chunks assert not with_chunks or publication_type == LAYER_TYPE + # Compress settings can be used only with compress option + assert not compress_settings or compress with app.app_context(): r_url = url_for(publication_type_def.post_workspace_publication_url, workspace=workspace) + temp_dir = None + if compress: + temp_dir = tempfile.mkdtemp(prefix="layman_zip_") + zip_file = util.compress_files(file_paths, compress_settings=compress_settings, output_dir=temp_dir) + file_paths = [zip_file] + files = [] try: data = {'name': name, @@ -309,6 +343,8 @@ def publish_workspace_publication(publication_type, file_paths, ) wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers) + if temp_dir: + shutil.rmtree(temp_dir) return response.json()[0] diff --git a/test_tools/util.py b/test_tools/util.py index 305e859e0..984ec351c 100644 --- a/test_tools/util.py +++ b/test_tools/util.py @@ -1,4 +1,6 @@ import time +from zipfile import ZipFile +import os import requests from requests.exceptions import ConnectionError from PIL import Image, ImageChops @@ -75,3 +77,20 @@ def assert_async_error(expected, thrown): expected.pop('http_code') for key, value in expected.items(): assert thrown[key] == value, f'key={key}, thrown_dict={thrown}, expected={expected}' + + +def compress_files(filepaths, *, compress_settings, output_dir): + file_name = (compress_settings.archive_name + if compress_settings and compress_settings.archive_name is not None + else 'temporary_zip_file') + '.zip' + inner_directory = compress_settings.inner_directory if compress_settings else None + inner_filename = compress_settings.file_name if compress_settings else None + zip_file = os.path.join(output_dir, file_name) + with ZipFile(zip_file, 'w') as zipfile: + for file in filepaths: + filename = os.path.split(file)[1] + _, ext = filename.split('.', 1) + final_filename = (inner_filename + '.' + ext) if inner_filename else filename + inner_path = os.path.join(inner_directory, final_filename) if inner_directory else final_filename + zipfile.write(file, arcname=inner_path) + return zip_file diff --git a/tests/dynamic_data/predefined_zip_files.py b/tests/dynamic_data/predefined_zip_files.py new file mode 100644 index 000000000..9ebf9e2be --- /dev/null +++ b/tests/dynamic_data/predefined_zip_files.py @@ -0,0 +1,43 @@ +from test_tools import process_client + +SMALL_LAYER_ZIP = { + 'file_paths': ['sample/layman.layer/small_layer.geojson'], + 'compress': True, +} + +NE_110M_ADMIN_0_BOUNDARY_LINES_LAND = { + 'file_paths': [ + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.cpg', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.dbf', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.prj', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.README.html', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.shp', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.shx', + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.VERSION.txt', + ], + 'compress': True, + 'compress_settings': process_client.CompressTypeDef(archive_name='ne_110m_admin_0_boundary lines land +ěščřžýáí', + inner_directory='/ne_110m_admin_0_boundary lines land +ěščřžýáí/', + file_name='ne_110m_admin_0_boundary_lines_land ížě', + ), +} + +SAMPLE_TIF_TFW_RGBA_OPAQUE = { + 'file_paths': [ + 'sample/layman.layer/sample_tif_tfw_rgba_opaque.tfw', + 'sample/layman.layer/sample_tif_tfw_rgba_opaque.tif', + ], + 'compress': True, + 'compress_settings': process_client.CompressTypeDef(inner_directory='/sample_tif_tfw_rgba_opaque/sample_tif_tfw_rgba_opaque/sample_tif_tfw_rgba_opaque/', + ), +} + +SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE = { + 'file_paths': [ + 'sample/layman.layer/sample_tif_colortable_nodata_opaque.tif', + 'sample/layman.layer/sample_tif_colortable_nodata_opaque.tif.aux.xml', + ], + 'compress': True, + 'compress_settings': process_client.CompressTypeDef(inner_directory='/sample_tif_colortable_nodata_opaque/', + ), +} diff --git a/tests/dynamic_data/publications.py b/tests/dynamic_data/publications.py index 0ed216f07..12cfa4304 100644 --- a/tests/dynamic_data/publications.py +++ b/tests/dynamic_data/publications.py @@ -1,7 +1,7 @@ import tests.asserts.final.publication as publication import tests.asserts.processing as processing from test_tools import process_client -from . import predefined_actions +from . import predefined_actions, predefined_zip_files from .. import Action, Publication, dynamic_data as consts @@ -55,7 +55,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/small_layer.zip'], + **predefined_zip_files.SMALL_LAYER_ZIP, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -81,7 +81,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'], + **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -93,7 +93,7 @@ 'exp_publication_detail': { 'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529], }, - 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp', + 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp', 'gdal_prefix': '/vsizip/', 'publ_type_detail': ('vector', 'sld'), }), @@ -107,7 +107,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_tfw_rgba_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_TFW_RGBA_OPAQUE, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -131,7 +131,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.patch_workspace_publication, { - 'file_paths': ['sample/layman.layer/small_layer.zip'], + **predefined_zip_files.SMALL_LAYER_ZIP, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -161,7 +161,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.patch_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -185,7 +185,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.patch_workspace_publication, { - 'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'], + **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -197,7 +197,7 @@ 'exp_publication_detail': { 'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529], }, - 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp', + 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp', 'gdal_prefix': '/vsizip/', 'publ_type_detail': ('vector', 'sld'), }), @@ -209,7 +209,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.patch_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -235,7 +235,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE, }), consts.KEY_RESPONSE_ASSERTS: [ Action(processing.response.valid_post, dict()), @@ -261,7 +261,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/small_layer.zip'], + **predefined_zip_files.SMALL_LAYER_ZIP, 'with_chunks': True, }), consts.KEY_RESPONSE_ASSERTS: [ @@ -288,7 +288,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'], + **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND, 'with_chunks': True, }), consts.KEY_RESPONSE_ASSERTS: [ @@ -301,7 +301,7 @@ 'exp_publication_detail': { 'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529], }, - 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp', + 'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp', 'gdal_prefix': '/vsizip/', 'publ_type_detail': ('vector', 'sld'), }), @@ -315,7 +315,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_tfw_rgba_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_TFW_RGBA_OPAQUE, 'with_chunks': True, }), consts.KEY_RESPONSE_ASSERTS: [ @@ -342,7 +342,7 @@ { consts.KEY_ACTION: { consts.KEY_CALL: Action(process_client.publish_workspace_publication, { - 'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'], + **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE, 'with_chunks': True, }), consts.KEY_RESPONSE_ASSERTS: [