diff --git a/sample/layman.layer/small_layer.zip b/sample/layman.layer/small_layer.zip new file mode 100644 index 000000000..92a7eb843 Binary files /dev/null and b/sample/layman.layer/small_layer.zip differ diff --git a/src/layman/layer/db/tasks.py b/src/layman/layer/db/tasks.py index 076a48c96..87ffd06c2 100644 --- a/src/layman/layer/db/tasks.py +++ b/src/layman/layer/db/tasks.py @@ -40,7 +40,7 @@ def refresh_table( if self.is_aborted(): raise AbortedException - main_filepath = get_layer_main_file_path(workspace, layername) + main_filepath = get_layer_main_file_path(workspace, layername, gdal_format=True) process = db.import_layer_vector_file_async(workspace, layername, main_filepath, crs_id) while process.poll() is None and not self.is_aborted(): pass diff --git a/src/layman/layer/filesystem/input_file.py b/src/layman/layer/filesystem/input_file.py index aa5141903..ffa4bedd8 100644 --- a/src/layman/layer/filesystem/input_file.py +++ b/src/layman/layer/filesystem/input_file.py @@ -35,11 +35,26 @@ def delete_layer(workspace, layername): util.delete_layer_subdir(workspace, layername, LAYER_SUBDIR) -def get_layer_info(workspace, layername): +def get_compressed_main_file_extension(filepath): + file_ext = os.path.splitext(filepath)[1] + return file_ext if file_ext in settings.COMPRESSED_FILE_EXTENSIONS else None + + +def get_layer_files(workspace, layername, *, only_physical_files=False): input_file_dir = get_layer_input_file_dir(workspace, layername) pattern = os.path.join(input_file_dir, layername + '.*') filepaths = glob.glob(pattern) - abs_main_filepath = get_main_file_name(filepaths) + if len(filepaths) == 1 and not only_physical_files: + compress_type = get_compressed_main_file_extension(filepaths[0]) + if compress_type: + compressed_filenames = util.get_filenames_from_zip_storage(filepaths[0]) + filepaths = [os.path.join(filepaths[0], fp) for fp in compressed_filenames] + return filepaths + + +def get_layer_info(workspace, layername): + abs_main_filepath = get_layer_main_file_path(workspace, layername, ) + if abs_main_filepath is not None: rel_main_filepath = os.path.relpath(abs_main_filepath, common_util.get_workspace_dir(workspace)) file_type = get_file_type(rel_main_filepath) @@ -76,11 +91,15 @@ def get_main_file_name(filenames): in get_all_allowed_main_extensions()), None) -def get_layer_main_file_path(workspace, layername): - input_file_dir = get_layer_input_file_dir(workspace, layername) - pattern = os.path.join(input_file_dir, layername + '.*') - filenames = glob.glob(pattern) - return get_main_file_name(filenames) +def get_layer_main_file_path(workspace, layername, *, gdal_format=False): + filepaths = get_layer_files(workspace, layername) + main_file = get_main_file_name(filepaths) + physical_files = get_layer_files(workspace, layername, only_physical_files=True) + if len(physical_files) == 1 and gdal_format: + compress_type = get_compressed_main_file_extension(physical_files[0]) + if compress_type: + main_file = settings.COMPRESSED_FILE_EXTENSIONS[compress_type] + main_file + return main_file def get_file_type(main_filepath): @@ -226,6 +245,15 @@ def save_layer_files(workspace, layername, files, check_crs, *, output_dir=None) check_layer_crs(filepath_mapping[main_filename]) +def save_layer_zip_file(workspace, layername, zip_file, *, output_dir=None): + output_dir = output_dir or ensure_layer_input_file_dir(workspace, layername) + _, filepath_mapping = get_file_name_mappings( + [zip_file.filename], zip_file.filename, layername, output_dir + ) + + common.save_files([zip_file], filepath_mapping) + + def get_unsafe_layername(files): filenames = list(map( lambda f: f if isinstance(f, str) else f.filename, diff --git a/src/layman/layer/filesystem/util.py b/src/layman/layer/filesystem/util.py index 89c5e3b0e..01ef7e9fe 100644 --- a/src/layman/layer/filesystem/util.py +++ b/src/layman/layer/filesystem/util.py @@ -1,4 +1,7 @@ +from zipfile import ZipFile from functools import partial +from werkzeug.datastructures import FileStorage + from layman.common.filesystem import util as publ_util LAYER_TYPE = '.'.join(__name__.split('.')[:-2]) @@ -14,3 +17,11 @@ # workspace, layername, subdir delete_layer_subdir = partial(publ_util.delete_publication_subdir, LAYER_TYPE) + + +def get_filenames_from_zip_storage(zip_file): + with ZipFile(zip_file) as opened_zip_file: + filenames = opened_zip_file.namelist() + if isinstance(zip_file, FileStorage): + zip_file.seek(0) + return filenames diff --git a/src/layman/layer/rest_workspace_layers.py b/src/layman/layer/rest_workspace_layers.py index d9cc79eff..0ea32e2ec 100644 --- a/src/layman/layer/rest_workspace_layers.py +++ b/src/layman/layer/rest_workspace_layers.py @@ -8,7 +8,7 @@ from layman.authz import authorize_workspace_publications_decorator from layman.common import redis as redis_util, rest as rest_common from . import util, LAYER_TYPE, LAYER_REST_PATH_NAME -from .filesystem import input_file, input_style, input_chunk, uuid +from .filesystem import input_file, input_style, input_chunk, uuid, util as fs_util bp = Blueprint('rest_workspace_layers', __name__) @@ -41,12 +41,15 @@ def post(workspace): # FILE use_chunk_upload = False + zipped_file = None files = [] if 'file' in request.files: files = [ f for f in request.files.getlist("file") if len(f.filename) > 0 ] + if len(files) == 1 and input_file.get_compressed_main_file_extension(files[0].filename): + zipped_file = files[0] if len(files) == 0 and len(request.form.getlist('file')) > 0: files = [ filename for filename in request.form.getlist('file') @@ -79,6 +82,8 @@ def post(workspace): # FILE NAMES if use_chunk_upload: filenames = files + elif zipped_file: + filenames = fs_util.get_filenames_from_zip_storage(zipped_file) else: filenames = [f.filename for f in files] file_type = input_file.get_file_type(input_file.get_main_file_name(filenames)) @@ -153,8 +158,14 @@ def post(workspace): }) else: try: - input_file.save_layer_files( - workspace, layername, files, check_crs) + if zipped_file: + input_file.save_layer_zip_file( + workspace, layername, zipped_file, + ) + else: + input_file.save_layer_files( + workspace, layername, files, check_crs + ) except BaseException as exc: uuid.delete_layer(workspace, layername) input_file.delete_layer(workspace, layername) diff --git a/src/layman_settings.py b/src/layman_settings.py index f0daf312c..89f3d8863 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -22,6 +22,11 @@ '.jpg': FILE_TYPE_RASTER, } +# Files are opened with dedicated tools for each format, so adding new extension is not sufficient for new compress format to start working +COMPRESSED_FILE_EXTENSIONS = { + '.zip': '/vsizip/', +} + INPUT_SRS_LIST = [ 'EPSG:3857', 'EPSG:4326', diff --git a/tests/asserts/final/publication/internal.py b/tests/asserts/final/publication/internal.py index 034bd6cae..d518477c5 100644 --- a/tests/asserts/final/publication/internal.py +++ b/tests/asserts/final/publication/internal.py @@ -90,6 +90,7 @@ def correct_values_in_detail(workspace, publ_type, name, exp_publication_detail) publ_type_dir = assert_util.get_directory_name_from_publ_type(publ_type) expected_detail = { 'name': name, + 'title': name, 'type': publ_type, 'thumbnail': { 'url': f'http://{settings.LAYMAN_PROXY_SERVER_NAME}/rest/workspaces/{workspace}/{publ_type_dir}/{name}/thumbnail', diff --git a/tests/dynamic_data/predefined_infos.py b/tests/dynamic_data/predefined_infos.py index a1949b163..cedf77ec1 100644 --- a/tests/dynamic_data/predefined_infos.py +++ b/tests/dynamic_data/predefined_infos.py @@ -1,5 +1,4 @@ BASIC_SLD_LAYER = { - 'title': 'basic_sld', 'style_type': 'sld', 'bounding_box': [1571204.369948366, 6268896.225570714, 1572590.854206196, 6269876.33561699], diff --git a/tests/dynamic_data/publications.py b/tests/dynamic_data/publications.py index 841718e71..f567809db 100644 --- a/tests/dynamic_data/publications.py +++ b/tests/dynamic_data/publications.py @@ -37,4 +37,30 @@ ], }, ], + Publication(consts.COMMON_WORKSPACE, consts.LAYER_TYPE, 'zipped_sld'): [ + { + consts.KEY_ACTION: { + consts.KEY_CALL: Action(process_client.publish_workspace_publication, { + 'file_paths': ['sample/layman.layer/small_layer.zip'], + }), + consts.KEY_RESPONSE_ASSERTS: [ + Action(processing.response.valid_post, dict()), + ], + }, + consts.KEY_FINAL_ASSERTS: [ + *publication.IS_LAYER_COMPLETE_AND_CONSISTENT, + Action(publication.internal.correct_values_in_detail, { + 'exp_publication_detail': { + **predefined_infos.BASIC_SLD_LAYER, + '_file': { + 'path': '/layman_data_test/workspaces/dynamic_test_workspace/layers/zipped_sld/input_file/zipped_sld.zip/small_layer.geojson' + }, + 'file': { + 'path': 'layers/zipped_sld/input_file/zipped_sld.zip/small_layer.geojson' + }, + }, + }), + ], + }, + ] }