Zip on the fly

CHANGELOG.md

@@ -103,7 +103,7 @@ make timgen-build
 - [#383](https://github.com/LayerManager/layman/issues/383) Add new Makefile target `upgrade-after-timeout` to finish upgrade in case of GeoServer call timeout.
 - Fix [GET Workspace Layer](doc/rest.md#get-workspace-layer) documentation; `style` item was incorrectly used instead of `sld`.
 - [#347](https://github.com/LayerManager/layman/issues/347) Upgrade PostgreSQL 10 to 13.3 and PostGIS 2.4 to 3.1. Use docker image from [layermanager/postgis@hub.docker.com](https://hub.docker.com/repository/docker/layermanager/postgis@github.com), source is located at [layermanager/docker-postgis@github.com](https://github.com/LayerManager/docker-postgis).
-- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/gdal/docker).
+- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/docker).
 - [#367](https://github.com/LayerManager/layman/issues/367) Upgrade also
   - python from 3.6 to 3.8
   - flask from 1.1 to 2.0

doc/dependencies.md

@@ -78,7 +78,7 @@
 | [postgis_restore.pl](https://github.com/postgis/postgis/blob/3.1.2/utils/postgis_restore.pl.in) | GNU GPL v2 | upgrade_v1_14_postgis_restore.pl | upgrade | src | |
 | [libxml2](http://xmlsoft.org/) | MIT | python3-lxml | prod | bin | |
 | [libxslt1.1](http://xmlsoft.org/libxslt/) | MIT | python3-lxml | prod | bin | |
-| [gdal/docker](https://github.com/OSGeo/gdal/tree/master/gdal/docker) | MIT License | Dockerfile | prod | bin | |
+| [gdal/docker](https://github.com/OSGeo/gdal/tree/master/docker) | MIT License | Dockerfile | prod | bin | |
 | [kartoza/docker-geoserver](https://github.com/kartoza/docker-geoserver) | GNU GPL v2 | docker-compose.yml | dev | bin | |
 | [jirikcz/qgis-server](https://github.com/LayerManager/docker-qgis-server) | GNU GPL v3 | docker-compose.yml | dev | bin | |
 | [layermanager/docker-postgis](https://github.com/layermanager/docker-postgis) | MIT | docker-compose.yml | dev | bin | |

test_tools/process_client.py

@@ -6,12 +6,15 @@
 from functools import partial
 from collections import namedtuple
 import xml.etree.ElementTree as ET
+import tempfile
+import shutil
 import requests
 
 from geoserver import error as gs_error
 from layman import app, settings, util as layman_util
 from layman.layer.geoserver import wfs, wms
 from layman.http import LaymanError
+from . import util
 from .util import url_for
 from .process import LAYMAN_CELERY_QUEUE
 
@@ -75,6 +78,13 @@
                                                         ),
                          }
 
+# pylint: disable=unexpected-keyword-arg
+CompressTypeDef = namedtuple('CompressTypeDef', [
+    'archive_name',
+    'inner_directory',
+    'file_name',
+], defaults=[None, None, None])
+
 
 def wait_for_rest(url, max_attempts, sleeping_time, check_response, headers=None):
     headers = headers or None
@@ -120,6 +130,7 @@ def upload_file_chunks(publication_type,
                             )
 
     file_chunks = [('file', file_name) for file_name in file_paths]
+    file_dict = None
     for file_type, file_name in file_chunks:
         try:
             basename = os.path.basename(file_name)
@@ -137,7 +148,8 @@ def upload_file_chunks(publication_type,
                                            data=data)
             raise_layman_error(chunk_response)
         finally:
-            file_dict[file_type][1].close()
+            if file_dict:
+                file_dict[file_type][1].close()
 
 
 def patch_workspace_publication(publication_type,
@@ -150,6 +162,8 @@ def patch_workspace_publication(publication_type,
                                 title=None,
                                 style_file=None,
                                 check_response_fn=None,
+                                compress=False,
+                                compress_settings=None,
                                 with_chunks=False,
                                 ):
     headers = headers or {}
@@ -160,12 +174,20 @@ def patch_workspace_publication(publication_type,
 
     # Only Layer files can be uploaded by chunks
     assert not with_chunks or publication_type == LAYER_TYPE
+    # Compress settings can be used only with compress option
+    assert not compress_settings or compress
 
     with app.app_context():
         r_url = url_for(publication_type_def.patch_workspace_publication_url,
                         workspace=workspace,
                         **{publication_type_def.url_param_name: name})
 
+    temp_dir = None
+    if compress:
+        temp_dir = tempfile.mkdtemp(prefix="layman_zip_")
+        zip_file = util.compress_files(file_paths, compress_settings=compress_settings, output_dir=temp_dir)
+        file_paths = [zip_file]
+
     for file_path in file_paths:
         assert os.path.isfile(file_path), file_path
     files = []
@@ -204,6 +226,8 @@ def patch_workspace_publication(publication_type,
     wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers)
     wfs.clear_cache(workspace)
     wms.clear_cache(workspace)
+    if temp_dir:
+        shutil.rmtree(temp_dir)
     return response.json()
 
 
@@ -254,6 +278,8 @@ def publish_workspace_publication(publication_type,
                                   description=None,
                                   check_response_fn=None,
                                   with_chunks=False,
+                                  compress=False,
+                                  compress_settings=None,
                                   crs=None,
                                   ):
     title = title or name
@@ -265,10 +291,18 @@ def publish_workspace_publication(publication_type,
 
     # Only Layer files can be uploaded by chunks
     assert not with_chunks or publication_type == LAYER_TYPE
+    # Compress settings can be used only with compress option
+    assert not compress_settings or compress
 
     with app.app_context():
         r_url = url_for(publication_type_def.post_workspace_publication_url, workspace=workspace)
 
+    temp_dir = None
+    if compress:
+        temp_dir = tempfile.mkdtemp(prefix="layman_zip_")
+        zip_file = util.compress_files(file_paths, compress_settings=compress_settings, output_dir=temp_dir)
+        file_paths = [zip_file]
+
     files = []
     try:
         data = {'name': name,
@@ -309,6 +343,8 @@ def publish_workspace_publication(publication_type,
                                file_paths, )
 
     wait_for_publication_status(workspace, publication_type, name, check_response_fn=check_response_fn, headers=headers)
+    if temp_dir:
+        shutil.rmtree(temp_dir)
     return response.json()[0]
 
 

test_tools/util.py

@@ -1,4 +1,6 @@
 import time
+from zipfile import ZipFile
+import os
 import requests
 from requests.exceptions import ConnectionError
 from PIL import Image, ImageChops
@@ -75,3 +77,20 @@ def assert_async_error(expected, thrown):
     expected.pop('http_code')
     for key, value in expected.items():
         assert thrown[key] == value, f'key={key}, thrown_dict={thrown}, expected={expected}'
+
+
+def compress_files(filepaths, *, compress_settings, output_dir):
+    file_name = (compress_settings.archive_name
+                 if compress_settings and compress_settings.archive_name is not None
+                 else 'temporary_zip_file') + '.zip'
+    inner_directory = compress_settings.inner_directory if compress_settings else None
+    inner_filename = compress_settings.file_name if compress_settings else None
+    zip_file = os.path.join(output_dir, file_name)
+    with ZipFile(zip_file, 'w') as zipfile:
+        for file in filepaths:
+            filename = os.path.split(file)[1]
+            _, ext = filename.split('.', 1)
+            final_filename = (inner_filename + '.' + ext) if inner_filename else filename
+            inner_path = os.path.join(inner_directory, final_filename) if inner_directory else final_filename
+            zipfile.write(file, arcname=inner_path)
+    return zip_file

tests/dynamic_data/predefined_zip_files.py

@@ -0,0 +1,43 @@
+from test_tools import process_client
+
+SMALL_LAYER_ZIP = {
+    'file_paths': ['sample/layman.layer/small_layer.geojson'],
+    'compress': True,
+}
+
+NE_110M_ADMIN_0_BOUNDARY_LINES_LAND = {
+    'file_paths': [
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.cpg',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.dbf',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.prj',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.README.html',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.shp',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.shx',
+        'tmp/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.VERSION.txt',
+    ],
+    'compress': True,
+    'compress_settings': process_client.CompressTypeDef(archive_name='ne_110m_admin_0_boundary lines land +ěščřžýáí',
+                                                        inner_directory='/ne_110m_admin_0_boundary lines land +ěščřžýáí/',
+                                                        file_name='ne_110m_admin_0_boundary_lines_land ížě',
+                                                        ),
+}
+
+SAMPLE_TIF_TFW_RGBA_OPAQUE = {
+    'file_paths': [
+        'sample/layman.layer/sample_tif_tfw_rgba_opaque.tfw',
+        'sample/layman.layer/sample_tif_tfw_rgba_opaque.tif',
+    ],
+    'compress': True,
+    'compress_settings': process_client.CompressTypeDef(inner_directory='/sample_tif_tfw_rgba_opaque/sample_tif_tfw_rgba_opaque/sample_tif_tfw_rgba_opaque/',
+                                                        ),
+}
+
+SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE = {
+    'file_paths': [
+        'sample/layman.layer/sample_tif_colortable_nodata_opaque.tif',
+        'sample/layman.layer/sample_tif_colortable_nodata_opaque.tif.aux.xml',
+    ],
+    'compress': True,
+    'compress_settings': process_client.CompressTypeDef(inner_directory='/sample_tif_colortable_nodata_opaque/',
+                                                        ),
+}

tests/dynamic_data/publications.py

@@ -1,7 +1,7 @@
 import tests.asserts.final.publication as publication
 import tests.asserts.processing as processing
 from test_tools import process_client
-from . import predefined_actions
+from . import predefined_actions, predefined_zip_files
 from .. import Action, Publication, dynamic_data as consts
 
 
@@ -55,7 +55,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/small_layer.zip'],
+                    **predefined_zip_files.SMALL_LAYER_ZIP,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -81,7 +81,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'],
+                    **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -93,7 +93,7 @@
                     'exp_publication_detail': {
                         'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529],
                     },
-                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp',
+                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp',
                     'gdal_prefix': '/vsizip/',
                     'publ_type_detail': ('vector', 'sld'),
                 }),
@@ -107,7 +107,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_tfw_rgba_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_TFW_RGBA_OPAQUE,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -131,7 +131,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.patch_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/small_layer.zip'],
+                    **predefined_zip_files.SMALL_LAYER_ZIP,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -161,7 +161,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.patch_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -185,7 +185,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.patch_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'],
+                    **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -197,7 +197,7 @@
                     'exp_publication_detail': {
                         'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529],
                     },
-                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp',
+                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp',
                     'gdal_prefix': '/vsizip/',
                     'publ_type_detail': ('vector', 'sld'),
                 }),
@@ -209,7 +209,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.patch_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -235,7 +235,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
                     Action(processing.response.valid_post, dict()),
@@ -261,7 +261,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/small_layer.zip'],
+                    **predefined_zip_files.SMALL_LAYER_ZIP,
                     'with_chunks': True,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
@@ -288,7 +288,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/ne_110m_admin_0_boundary lines land +ěščřžýáí.zip'],
+                    **predefined_zip_files.NE_110M_ADMIN_0_BOUNDARY_LINES_LAND,
                     'with_chunks': True,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
@@ -301,7 +301,7 @@
                     'exp_publication_detail': {
                         'bounding_box': [-15695801.072582014, -7341864.739114417, 15699816.562538767, 11122367.192100529],
                     },
-                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin 0 boundary_lines_land ížě.shp',
+                    'file_extension': 'zip/ne_110m_admin_0_boundary lines land +ěščřžýáí/ne_110m_admin_0_boundary_lines_land ížě.shp',
                     'gdal_prefix': '/vsizip/',
                     'publ_type_detail': ('vector', 'sld'),
                 }),
@@ -315,7 +315,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_tfw_rgba_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_TFW_RGBA_OPAQUE,
                     'with_chunks': True,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [
@@ -342,7 +342,7 @@
         {
             consts.KEY_ACTION: {
                 consts.KEY_CALL: Action(process_client.publish_workspace_publication, {
-                    'file_paths': ['sample/layman.layer/sample_tif_colortable_nodata_opaque.zip'],
+                    **predefined_zip_files.SAMPLE_TIF_COLORTABLE_NODATA_OPAQUE,
                     'with_chunks': True,
                 }),
                 consts.KEY_RESPONSE_ASSERTS: [