From c626c1f3fc3b6129033582a9ab05167a978b7cf4 Mon Sep 17 00:00:00 2001 From: jirik Date: Sat, 7 Dec 2019 14:11:43 +0100 Subject: [PATCH] Set proxied server name, async generate metadata file of layer --- .env.demo | 1 + .env.dev | 1 + .env.test | 1 + src/layman/__init__.py | 1 + src/layman/{ => common}/metadata/__init__.py | 0 src/layman/{ => common}/metadata/util.py | 26 +- src/layman/{ => common}/metadata/util_test.py | 22 +- .../metadata/util_test_filled_template.xml | 260 ++++++++++++++++++ src/layman/layer/__init__.py | 1 + .../filesystem/metadata-template.xml} | 0 .../filesystem/metadata.py} | 75 ++++- src/layman/layer/filesystem/tasks.py | 21 +- src/layman/layer/micka/__init__.py | 14 + src/layman_settings_demo.py | 1 + src/layman_settings_dev.py | 1 + src/layman_settings_test.py | 1 + 16 files changed, 410 insertions(+), 16 deletions(-) rename src/layman/{ => common}/metadata/__init__.py (100%) rename src/layman/{ => common}/metadata/util.py (70%) rename src/layman/{ => common}/metadata/util_test.py (60%) create mode 100644 src/layman/common/metadata/util_test_filled_template.xml rename src/layman/{metadata/layer-template.xml => layer/filesystem/metadata-template.xml} (100%) rename src/layman/{metadata/template.py => layer/filesystem/metadata.py} (68%) create mode 100644 src/layman/layer/micka/__init__.py diff --git a/.env.demo b/.env.demo index 4f2b9187d..33d942855 100644 --- a/.env.demo +++ b/.env.demo @@ -38,6 +38,7 @@ LAYMAN_SKIP_REDIS_LOADING=false # Flask-style server name (domain and port) LAYMAN_SERVER_NAME=layman:8000 +LAYMAN_PROXY_SERVER_NAME=layman:8000 # internal settings LAYMAN_TIMGEN_URL=http://hslayers:8080/ diff --git a/.env.dev b/.env.dev index 79beec0cd..50be9d9e7 100644 --- a/.env.dev +++ b/.env.dev @@ -39,6 +39,7 @@ LAYMAN_SKIP_REDIS_LOADING=false # Flask-style server name (domain and port) LAYMAN_SERVER_NAME=layman_dev:8000 +LAYMAN_PROXY_SERVER_NAME=layman_dev:8000 # internal settings LAYMAN_TIMGEN_URL=http://hslayers:8080/ diff --git a/.env.test b/.env.test index d495b2c77..65cd9cbe2 100644 --- a/.env.test +++ b/.env.test @@ -39,6 +39,7 @@ LAYMAN_SKIP_REDIS_LOADING=false # Flask-style server name (domain and port) LAYMAN_SERVER_NAME=layman_test_run_1:8000 +LAYMAN_PROXY_SERVER_NAME=layman_test_run_1:8000 # internal settings LAYMAN_TIMGEN_URL=http://hslayers:8080/ diff --git a/src/layman/__init__.py b/src/layman/__init__.py index fc57a64e9..3509fa0a9 100644 --- a/src/layman/__init__.py +++ b/src/layman/__init__.py @@ -7,6 +7,7 @@ app = Flask(__name__) app.secret_key = os.environ['FLASK_SECRET_KEY'] +app.config['SERVER_NAME'] = settings.LAYMAN_PROXY_SERVER_NAME from .http import LaymanError diff --git a/src/layman/metadata/__init__.py b/src/layman/common/metadata/__init__.py similarity index 100% rename from src/layman/metadata/__init__.py rename to src/layman/common/metadata/__init__.py diff --git a/src/layman/metadata/util.py b/src/layman/common/metadata/util.py similarity index 70% rename from src/layman/metadata/util.py rename to src/layman/common/metadata/util.py index 773852298..3deb6b93f 100644 --- a/src/layman/metadata/util.py +++ b/src/layman/common/metadata/util.py @@ -1,5 +1,5 @@ -from . import template from xml.etree import ElementTree as ET +from io import BytesIO NAMESPACES = { @@ -28,21 +28,23 @@ def indent(elem, level=0): elem.tail = i -def fill_template(template_path, filled_path=None): +def fill_template(template_path, template_values): with open(template_path, 'r') as template_file: template_str = template_file.read() - defaults = template.get_layer_values() - xml_str = template_str.format(**defaults) + xml_str = template_str.format(**template_values) root_el = ET.fromstring(xml_str) - # for k, v in namespace_map.items(): - # root_el.attrib[f"xmlns:{k}"] = v indent(root_el) el_tree = ET.ElementTree(root_el) - if filled_path is not None: - el_tree.write( - filled_path, - encoding='UTF-8', - xml_declaration=True, - ) + file_object = BytesIO() + + el_tree.write( + file_object, + encoding='UTF-8', + xml_declaration=True, + ) + + file_object.seek(0) + + return file_object diff --git a/src/layman/metadata/util_test.py b/src/layman/common/metadata/util_test.py similarity index 60% rename from src/layman/metadata/util_test.py rename to src/layman/common/metadata/util_test.py index 31721cceb..03bee41c6 100644 --- a/src/layman/metadata/util_test.py +++ b/src/layman/common/metadata/util_test.py @@ -1,12 +1,16 @@ from multiprocessing import Process import pytest import time +import os +import filecmp +import difflib import sys del sys.modules['layman'] from layman import app as app from layman import settings +from layman.layer.filesystem.metadata import _get_template_values from . import util @@ -47,4 +51,20 @@ def app_context(): @pytest.mark.usefixtures('app_context') def test_fill_template(client): - xmlstr = util.fill_template('src/layman/metadata/layer-template.xml', filled_path='src/layman/metadata/INSPIRE2-min.xml') + xml_path = 'tmp/metadata-template.xml' + try: + os.remove(xml_path) + except OSError: + pass + file_object = util.fill_template('src/layman/layer/filesystem/metadata-template.xml', _get_template_values()) + with open(xml_path, 'wb') as out: + out.write(file_object.read()) + + def get_diff(p1, p2): + diff = difflib.unified_diff(open(p1).readlines(), open(p2).readlines()) + return f"diff={''.join(diff)}" + + expected_path = 'src/layman/common/metadata/util_test_filled_template.xml' + assert filecmp.cmp(xml_path, expected_path), get_diff(xml_path, expected_path) + + diff --git a/src/layman/common/metadata/util_test_filled_template.xml b/src/layman/common/metadata/util_test_filled_template.xml new file mode 100644 index 000000000..c39d67db3 --- /dev/null +++ b/src/layman/common/metadata/util_test_filled_template.xml @@ -0,0 +1,260 @@ + + + + mca238200-8200-1a23-9399-42c9fca53542 + + + cze + + + dataset + + + + + + + + + + + + + + + + + + + + pointOfContact + + + + + + + + + + EPSG:3857 + + + + + + + + + + + EPSG:4326 + + + + + + + + + + + CORINE - Krajinný pokryv CLC 90 + + + + + 2007-05-25 + + + revision + + + + + + + MZP-CORINE + + + + + + + + + + + + + + + + + + + + + + + + + custodian + + + + + + + http://layman_test_run_1:8000/rest/browser/layers/layer/thumbnail + + + PNG + + + + + + + + + + GEMET - INSPIRE themes, version 1.0 + + + + + 2008-06-01 + + + publication + + + + + + + + + + + otherRestrictions + + + Bez omezení + + + + + + + otherRestrictions + + + Žádné podmínky neplatí + + + + + vector + + + + + + + + + + + + + utf8 + + + + + + + + 11.87 + + + 19.13 + + + 48.12 + + + 51.59 + + + + + + + + + + + + + + + http://www.env.cz/corine/data/download.zip + + + OGC:WMS-1.3.0 + + + download + + + + + + + http://www.env.cz/corine/data/download.zip + + + OGC:WFS-2.0.0 + + + download + + + + + + + http://layman_test_run_1:8000/rest/browser/layers/layer + + + WWW:LINK-1.0-http--link + + + information + + + + + + + + + + + + + dataset + + + + + + + + + + + diff --git a/src/layman/layer/__init__.py b/src/layman/layer/__init__.py index 8332454b7..91f5c2d06 100644 --- a/src/layman/layer/__init__.py +++ b/src/layman/layer/__init__.py @@ -35,6 +35,7 @@ def get_layer_sources(): 'layman.layer.geoserver.wms', 'layman.layer.geoserver.sld', 'layman.layer.filesystem.thumbnail', + 'layman.layer.filesystem.metadata', ], 'task_modules': [ 'layman.layer.db.tasks', diff --git a/src/layman/metadata/layer-template.xml b/src/layman/layer/filesystem/metadata-template.xml similarity index 100% rename from src/layman/metadata/layer-template.xml rename to src/layman/layer/filesystem/metadata-template.xml diff --git a/src/layman/metadata/template.py b/src/layman/layer/filesystem/metadata.py similarity index 68% rename from src/layman/metadata/template.py rename to src/layman/layer/filesystem/metadata.py index 75a9e9ee2..5e9d774b2 100644 --- a/src/layman/metadata/template.py +++ b/src/layman/layer/filesystem/metadata.py @@ -1,7 +1,78 @@ -from flask import url_for +import os +import pathlib +from flask import url_for, current_app -def get_layer_values( +from . import util +from . import input_file +from layman.common.metadata.util import fill_template + + +DIR = __name__.split('.')[-1] + + +def get_dir(username, layername): + input_sld_dir = os.path.join(util.get_layer_dir(username, layername), + DIR) + return input_sld_dir + + +def ensure_dir(username, layername): + input_sld_dir = get_dir(username, layername) + pathlib.Path(input_sld_dir).mkdir(parents=True, exist_ok=True) + return input_sld_dir + + +get_layer_info = input_file.get_layer_info + + +get_layer_names = input_file.get_layer_names + + +update_layer = input_file.update_layer + + +get_publication_names = input_file.get_publication_names + + +get_publication_uuid = input_file.get_publication_uuid + + +def delete_layer(username, layername): + util.delete_layer_subdir(username, layername, DIR) + + +def get_file_path(username, layername): + input_sld_dir = get_dir(username, layername) + return os.path.join(input_sld_dir, layername+'.xml') + + +def save_file(username, layername, xml_file): + xml_path = get_file_path(username, layername) + if xml_file is None: + delete_layer(username, layername) + else: + ensure_dir(username, layername) + with open(xml_path, 'wb') as out: + out.write(xml_file.read()) + + +def get_file(username, layername): + sld_path = get_file_path(username, layername) + + if os.path.exists(sld_path): + return open(sld_path, 'rb') + return None + + +def create_file(username, layername): + template_values = _get_template_values(username=username, layername=layername) + template_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'metadata-template.xml') + file_object = fill_template(template_path, template_values) + save_file(username, layername, file_object) + + +def _get_template_values( uuid='ca238200-8200-1a23-9399-42c9fca53542', epsg_codes=None, title='CORINE - Krajinný pokryv CLC 90', diff --git a/src/layman/layer/filesystem/tasks.py b/src/layman/layer/filesystem/tasks.py index 812675a8b..13e1764a0 100644 --- a/src/layman/layer/filesystem/tasks.py +++ b/src/layman/layer/filesystem/tasks.py @@ -6,7 +6,7 @@ from layman import celery_app from layman.http import LaymanError from layman import settings -from . import input_file, input_chunk, thumbnail +from . import input_file, input_chunk, thumbnail, metadata logger = get_task_logger(__name__) @@ -75,3 +75,22 @@ def refresh_thumbnail(self, username, layername): thumbnail.delete_layer(username, layername) raise AbortedException + +def refresh_metadata_needed(username, layername, task_options): + return True + + +@celery_app.task( + name='layman.layer.filesystem.metadata.refresh', + bind=True, + base=celery_app.AbortableTask +) +def refresh_metadata(self, username, layername): + if self.is_aborted(): + raise AbortedException + metadata.create_file(username, layername) + + if self.is_aborted(): + metadata.delete_layer(username, layername) + raise AbortedException + diff --git a/src/layman/layer/micka/__init__.py b/src/layman/layer/micka/__init__.py new file mode 100644 index 000000000..de018c823 --- /dev/null +++ b/src/layman/layer/micka/__init__.py @@ -0,0 +1,14 @@ +from layman.common.filesystem import util as common_util + + +get_usernames = common_util.get_usernames +ensure_user_workspace = common_util.ensure_user_workspace +delete_user_workspace = common_util.delete_user_workspace + + +def check_username(username): + pass + + +def check_new_layername(username, layername): + pass \ No newline at end of file diff --git a/src/layman_settings_demo.py b/src/layman_settings_demo.py index 9fa25fc8a..8adafe889 100644 --- a/src/layman_settings_demo.py +++ b/src/layman_settings_demo.py @@ -125,3 +125,4 @@ LAYMAN_CLIENT_URL = os.environ['LAYMAN_CLIENT_URL'] LAYMAN_CLIENT_PUBLIC_URL = os.getenv('LAYMAN_CLIENT_PUBLIC_URL', None) LAYMAN_SERVER_NAME = os.environ['LAYMAN_SERVER_NAME'] +LAYMAN_PROXY_SERVER_NAME = os.environ['LAYMAN_PROXY_SERVER_NAME'] diff --git a/src/layman_settings_dev.py b/src/layman_settings_dev.py index 9fa25fc8a..8adafe889 100644 --- a/src/layman_settings_dev.py +++ b/src/layman_settings_dev.py @@ -125,3 +125,4 @@ LAYMAN_CLIENT_URL = os.environ['LAYMAN_CLIENT_URL'] LAYMAN_CLIENT_PUBLIC_URL = os.getenv('LAYMAN_CLIENT_PUBLIC_URL', None) LAYMAN_SERVER_NAME = os.environ['LAYMAN_SERVER_NAME'] +LAYMAN_PROXY_SERVER_NAME = os.environ['LAYMAN_PROXY_SERVER_NAME'] diff --git a/src/layman_settings_test.py b/src/layman_settings_test.py index 9fa25fc8a..8adafe889 100644 --- a/src/layman_settings_test.py +++ b/src/layman_settings_test.py @@ -125,3 +125,4 @@ LAYMAN_CLIENT_URL = os.environ['LAYMAN_CLIENT_URL'] LAYMAN_CLIENT_PUBLIC_URL = os.getenv('LAYMAN_CLIENT_PUBLIC_URL', None) LAYMAN_SERVER_NAME = os.environ['LAYMAN_SERVER_NAME'] +LAYMAN_PROXY_SERVER_NAME = os.environ['LAYMAN_PROXY_SERVER_NAME']