diff --git a/CHANGELOG.md b/CHANGELOG.md index b7cf2751f..53223df67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Changes - [#65](https://github.com/jirik/layman/issues/65) [WFS endpoint](doc/rest.md#get-layer) accepts same [authentication](doc/security.md#authentication) credentials (e.g. [OAuth2 headers](doc/oauth2/index.md#request-layman-rest-api)) as Layman REST API endpoints. It's implemented using Layman's WFS proxy. This proxy authenticates the user and send user identification to GeoServer. In combination with changes in v1.6.0, Layman's [`read-everyone-write-owner` authorization](doc/security.md#authorization) (when active) is propagated to GeoServer and user can change only hers layers. - [#65](https://github.com/jirik/layman/issues/65) Layman automatically setup [HTTP authentication attribute](https://docs.geoserver.org/stable/en/user/security/tutorials/httpheaderproxy/index.html) and chain filter at startup. Secret value of this attribute can be changed in [LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE](doc/env-settings.md#LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE) and is used by Layman's WFS proxy. +- [#95](https://github.com/jirik/layman/issues/95) When calling WFS Transaction, Layman will automatically create missing attributes in DB before redirecting request to GeoServer. Each missing attribute is created as `VARCHAR(1024)`. Works for WFS-T 1.0, 1.1 and 2.0, actions Insert, Update and Replace. If creating attribute fails for any reason, warning is logged and request is redirected nevertheless. ## v1.6.1 2020-08-19 diff --git a/README.md b/README.md index 30fd9e560..b7c316b10 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,10 @@ Publishing geospatial data online through [REST API](doc/rest.md). - Asynchronous processing - Each vector dataset is automatically imported into PostGIS database - Provides URL endpoints - - [Web Map Service (WMS)](https://www.opengeospatial.org/standards/wms) - - [Web Feature Service (WFS)](https://www.opengeospatial.org/standards/wfs) - - [Catalogue Service (CSW)](https://www.opengeospatial.org/standards/cat) - - thumbnail image -- Documented [REST API](doc/rest.md) + - [REST API](doc/rest.md) + - [Web Map Service (WMS)](doc/endpoints.md#web-map-service) + - [Web Feature Service (WFS)](doc/endpoints.md#web-feature-service) + - [Catalogue Service (CSW)](doc/endpoints.md#catalogue-service) - Documented [security system](doc/security.md) - Documented [data storage](doc/data-storage.md) - Configurable by environment variables diff --git a/doc/endpoints.md b/doc/endpoints.md new file mode 100644 index 000000000..c62d70881 --- /dev/null +++ b/doc/endpoints.md @@ -0,0 +1,15 @@ +# Endpoints + +## Web Map Service +[Web Map Service (WMS)](https://www.opengeospatial.org/standards/wms) endpoint is implemented using [GeoServer](https://docs.geoserver.org/2.13.0/user/services/wms/reference.html). No additional functionality is added by Layman. + + +## Web Feature Service +[Web Map Service (WMS)](https://www.opengeospatial.org/standards/wms) endpoint is implemented using combination of Layman's proxy and [GeoServer](https://docs.geoserver.org/2.13.0/user/services/wfs/reference.html). + +Layman's proxy has two functions: + - Understands same [authentication credentials](security.md#authentication) as Layman REST API (e.g. OAuth2 credentials) and passes authenticated user to GeoServer + - Creates missing attributes in DB before redirecting WFS-T request to GeoServer. Each missing attribute is created as `VARCHAR(1024)`. Works for WFS-T 1.0, 1.1 and 2.0, actions Insert, Update and Replace. If creating attribute fails for any reason, warning is logged and request is redirected nevertheless. + +## Catalogue Service +[Catalogue Service (CSW)](https://www.opengeospatial.org/standards/cat) is implemented using [Micka](https://github.com/hsrs-cz/Micka). diff --git a/src/layman/authz/read_everyone_write_everyone.py b/src/layman/authz/read_everyone_write_everyone.py index 5871602d3..ff8fc7d95 100644 --- a/src/layman/authz/read_everyone_write_everyone.py +++ b/src/layman/authz/read_everyone_write_everyone.py @@ -22,3 +22,7 @@ def get_gs_roles(username, type): elif type == 'w': roles = gs.get_roles_anyone(username) return roles + + +def can_i_edit(publ_type, username, publication_name): + return True diff --git a/src/layman/authz/read_everyone_write_owner.py b/src/layman/authz/read_everyone_write_owner.py index c7c4a527a..a38acca1e 100644 --- a/src/layman/authz/read_everyone_write_owner.py +++ b/src/layman/authz/read_everyone_write_owner.py @@ -43,3 +43,7 @@ def get_gs_roles(username, type): elif type == 'w': roles = gs.get_roles_owner(username) return roles + + +def can_i_edit(publ_type, username, publication_name): + return g.user is not None and g.user['username'] == username diff --git a/src/layman/common/geoserver/__init__.py b/src/layman/common/geoserver/__init__.py index f8b139937..5d4bbd123 100644 --- a/src/layman/common/geoserver/__init__.py +++ b/src/layman/common/geoserver/__init__.py @@ -467,3 +467,13 @@ def get_roles_anyone(username): def get_roles_owner(username): roles = {username_to_rolename(username)} return roles + + +def reset(auth): + logger.info(f"Resetting GeoServer") + r_url = settings.LAYMAN_GS_REST + 'reset' + r = requests.post(r_url, + headers=headers_json, + auth=auth + ) + r.raise_for_status() diff --git a/src/layman/gs_wfs_proxy.py b/src/layman/gs_wfs_proxy.py index c3da409ee..15a0460ef 100644 --- a/src/layman/gs_wfs_proxy.py +++ b/src/layman/gs_wfs_proxy.py @@ -1,9 +1,20 @@ +import re +import traceback + import requests +from lxml import etree as ET from flask import Blueprint, g, current_app as app, request, Response from layman.authn import authenticate from layman import settings +from layman.layer import db +from layman.layer.util import LAYERNAME_RE, ATTRNAME_RE +from layman.util import USERNAME_ONLY_PATTERN +from layman.common.geoserver import reset as gs_reset +from layman.layer import LAYER_TYPE +from layman.authz import util as authz + bp = Blueprint('gs_wfs_proxy_bp', __name__) @@ -14,19 +25,136 @@ def before_request(): pass +def ensure_wfs_t_attributes(binary_data): + try: + xml_tree = ET.XML(binary_data) + version = xml_tree.get('version')[0:4] + if version not in ["2.0.", "1.0.", "1.1."] or xml_tree.get('service').upper() != "WFS": + app.logger.warning(f"WFS Proxy: only xml version 2.0, 1.1, 1.0 and WFS service are supported. Request " + f"only redirected. Version={xml_tree.get('version')}, service={xml_tree.get('service')}") + return + + authz_module = authz.get_authz_module() + attribs = set() + for action in xml_tree: + action_qname = ET.QName(action) + if action_qname.localname in ('Insert', 'Replace'): + extracted_attribs = extract_attributes_from_wfs_t_insert_replace(action, authz_module) + attribs.update(extracted_attribs) + elif action_qname.localname in ('Update'): + extracted_attribs = extract_attributes_from_wfs_t_update(action, + authz_module, + xml_tree, + major_version=version[0:1]) + attribs.update(extracted_attribs) + + app.logger.info(f"GET WFS check_xml_for_attribute attribs={attribs}") + if attribs: + created_attributes = db.ensure_attributes(attribs) + if created_attributes: + gs_reset(settings.LAYMAN_GS_AUTH) + + except BaseException as err: + app.logger.warning(f"WFS Proxy: error={err}, trace={traceback.format_exc()}") + + +def extract_attributes_from_wfs_t_update(action, authz_module, xml_tree, major_version="2"): + attribs = set() + layer_qname = action.get('typeName').split(':') + ws_namespace = layer_qname[0] + ws_match = re.match(r"^(" + USERNAME_ONLY_PATTERN + ")$", ws_namespace) + if ws_match: + ws_name = ws_match.group(1) + else: + app.logger.warning(f"WFS Proxy: skipping due to wrong namespace name. Namespace={ws_namespace}") + return attribs + layer_name = layer_qname[1] + layer_match = re.match(LAYERNAME_RE, layer_name) + if not layer_match: + app.logger.warning(f"WFS Proxy: skipping due to wrong layer name. Layer name={layer_name}") + return attribs + if not authz_module.can_i_edit(LAYER_TYPE, ws_name, layer_name): + app.logger.warning(f"Can not edit. ws_namespace={ws_namespace}") + return attribs + value_ref_string = "Name" if major_version == "1" else "ValueReference" + properties = action.xpath('wfs:Property/wfs:' + value_ref_string, namespaces=xml_tree.nsmap) + for prop in properties: + split_text = prop.text.split(':') + # No namespace in element text + if len(split_text) == 1: + attrib_name = split_text[0] + # There is namespace in element text + elif len(split_text) == 2: + if split_text[0] != ws_namespace: + app.logger.warning(f"WFS Proxy: skipping due to different namespace in layer and in " + f"property. Layer namespace={ws_namespace}, " + f"property namespace={split_text[0]}") + continue + attrib_name = split_text[1] + attrib_match = re.match(ATTRNAME_RE, attrib_name) + if not attrib_match: + app.logger.warning(f"WFS Proxy: skipping due to wrong attribute name. " + f"Property={attrib_name}") + continue + attribs.add((ws_name, + layer_name, + attrib_name)) + return attribs + + +def extract_attributes_from_wfs_t_insert_replace(action, authz_module): + attribs = set() + for layer in action: + layer_qname = ET.QName(layer) + ws_namespace = layer_qname.namespace + ws_match = re.match(r"^http://(" + USERNAME_ONLY_PATTERN + ")$", ws_namespace) + if ws_match: + ws_name = ws_match.group(1) + else: + app.logger.warning(f"WFS Proxy: skipping due to wrong namespace name. Namespace={ws_namespace}") + continue + layer_name = layer_qname.localname + layer_match = re.match(LAYERNAME_RE, layer_name) + if not layer_match: + app.logger.warning(f"WFS Proxy: skipping due to wrong layer name. Layer name={layer_name}") + continue + if not authz_module.can_i_edit(LAYER_TYPE, ws_name, layer_name): + app.logger.warning(f"WFS Proxy: Can not edit. ws_namespace={ws_name}") + continue + for attrib in layer: + attrib_qname = ET.QName(attrib) + if attrib_qname.namespace != ws_namespace: + app.logger.warning(f"WFS Proxy: skipping due to different namespace in layer and in " + f"property. Layer namespace={ws_namespace}, " + f"property namespace={attrib_qname.namespace}") + continue + attrib_name = attrib_qname.localname + attrib_match = re.match(ATTRNAME_RE, attrib_name) + if not attrib_match: + app.logger.warning(f"WFS Proxy: skipping due to wrong property name. Property name={attrib_name}") + continue + attribs.add((ws_name, + layer_name, + attrib_name)) + return attribs + + @bp.route('/', methods=['POST', 'GET']) def proxy(subpath): app.logger.info(f"GET WFS proxy, user={g.user}, subpath={subpath}, url={request.url}, request.query_string={request.query_string.decode('UTF-8')}") url = settings.LAYMAN_GS_URL + subpath + '?' + request.query_string.decode('UTF-8') headers_req = {key.lower(): value for (key, value) in request.headers if key.lower() not in ['host', settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE.lower()]} + data = request.get_data() if g.user is not None: headers_req[settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE] = g.user['username'] app.logger.info(f"GET WFS proxy, headers_req={headers_req}, url={url}") + if data is not None and len(data) > 0: + ensure_wfs_t_attributes(data) response = requests.request(method=request.method, url=url, - data=request.get_data(), + data=data, headers=headers_req, cookies=request.cookies, allow_redirects=False diff --git a/src/layman/gs_wfs_proxy_test.py b/src/layman/gs_wfs_proxy_test.py index 2c37c0f39..4f26667fd 100644 --- a/src/layman/gs_wfs_proxy_test.py +++ b/src/layman/gs_wfs_proxy_test.py @@ -3,6 +3,7 @@ import time from flask import url_for import pytest +from urllib.parse import urljoin import sys import os @@ -12,7 +13,10 @@ from layman import app from layman import settings from layman.layer.rest_test import wait_till_ready +from layman.layer import db from test import process, client as client_util +from test.data import wfs as data_wfs +from layman.layer.geoserver.util import wfs_proxy liferay_mock = process.liferay_mock @@ -55,29 +59,7 @@ def test_rest_get(client): username = 'wfs_proxy_test' layername = 'layer_wfs_proxy_test' - with app.app_context(): - rest_path = url_for('rest_layers.post', username=username) - - file_paths = [ - 'tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson', - ] - - for fp in file_paths: - assert os.path.isfile(fp) - files = [] - - try: - files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] - rv = client.post(rest_path, data={ - 'file': files, - 'name': layername - }) - assert rv.status_code == 200 - finally: - for fp in files: - fp[0].close() - - wait_till_ready(username, layername) + setup_layer_flask(username, layername, client) rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{username}/wfs?request=Transaction" headers = { @@ -85,7 +67,7 @@ def test_rest_get(client): 'Content-type': 'text/xml', } - data_xml = client_util.get_wfs_insert_points(username, layername) + data_xml = data_wfs.get_wfs20_insert_points(username, layername) with app.app_context(): r = client.post(rest_url, @@ -102,7 +84,22 @@ def test_rest_get(client): with app.app_context(): rest_path = url_for('rest_layer.delete_layer', username=username, layername=layername) client.delete(rest_path) - assert rv.status_code == 200 + assert r.status_code == 200 + + +def get_auth_header(username): + return {f'{ISS_URL_HEADER}': 'http://localhost:8082/o/oauth2/authorize', + f'{TOKEN_HEADER}': f'Bearer {username}', + } + + +def setup_user_layer(username, layername, authn_headers): + client_util.reserve_username(username, headers=authn_headers) + ln = client_util.publish_layer(username, layername, [ + 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson', + ], headers=authn_headers) + + assert ln == layername def test_wfs_proxy(liferay_mock): @@ -114,17 +111,9 @@ def test_wfs_proxy(liferay_mock): 'LAYMAN_AUTHZ_MODULE': 'layman.authz.read_everyone_write_owner', }, **AUTHN_SETTINGS)) - authn_headers1 = { - f'{ISS_URL_HEADER}': 'http://localhost:8082/o/oauth2/authorize', - f'{TOKEN_HEADER}': f'Bearer {username}', - } - - client_util.reserve_username(username, headers=authn_headers1) - ln = client_util.publish_layer(username, layername1, [ - 'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson', - ], headers=authn_headers1) + authn_headers1 = get_auth_header(username) - assert ln == layername1 + setup_user_layer(username, layername1, authn_headers1) rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{username}/wfs?request=Transaction" headers = { @@ -133,7 +122,7 @@ def test_wfs_proxy(liferay_mock): **authn_headers1, } - data_xml = client_util.get_wfs_insert_points(username, layername1) + data_xml = data_wfs.get_wfs20_insert_points(username, layername1) r = requests.post(rest_url, data=data_xml, @@ -148,10 +137,7 @@ def test_wfs_proxy(liferay_mock): assert r.status_code == 200, r.text # Testing, that user2 is not able to write to layer of user1 - authn_headers2 = { - f'{ISS_URL_HEADER}': 'http://localhost:8082/o/oauth2/authorize', - f'{TOKEN_HEADER}': f'Bearer {username2}', - } + authn_headers2 = get_auth_header(username2) headers2 = { 'Accept': 'text/xml', @@ -196,4 +182,212 @@ def test_wfs_proxy(liferay_mock): headers=headers4) assert r.status_code == 400, r.text + client_util.delete_layer(username, layername1, headers) + + process.stop_process(layman_process) + + +def setup_layer_flask(username, layername, client): + with app.app_context(): + rest_path = url_for('rest_layers.post', username=username) + + file_paths = [ + 'tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson', + ] + + for fp in file_paths: + assert os.path.isfile(fp) + files = [] + + try: + files = [(open(fp, 'rb'), os.path.basename(fp)) for fp in file_paths] + rv = client.post(rest_path, data={ + 'file': files, + 'name': layername + }) + assert rv.status_code == 200 + finally: + for fp in files: + fp[0].close() + + wait_till_ready(username, layername) + + +def test_missing_attribute(client): + username = 'testmissingattr' + layername = 'inexisting_attribute_layer' + layername2 = 'inexisting_attribute_layer2' + + setup_layer_flask(username, layername, client) + setup_layer_flask(username, layername2, client) + + rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/wfs?request=Transaction" + headers = { + 'Accept': 'text/xml', + 'Content-type': 'text/xml', + } + + def wfs_post(username, attr_names_list, data_xml): + with app.app_context(): + wfs_url = urljoin(settings.LAYMAN_GS_URL, username + '/ows') + old_db_attributes = {} + old_wfs_properties = {} + for layername, attr_names in attr_names_list: + # test that all attr_names are not yet presented in DB table + old_db_attributes[layername] = db.get_all_column_names(username, layername) + for attr_name in attr_names: + assert attr_name not in old_db_attributes[layername], f"old_db_attributes={old_db_attributes[layername]}, attr_name={attr_name}" + wfs = wfs_proxy(wfs_url) + layer_schema = wfs.get_schema(f"{username}:{layername}") + old_wfs_properties[layername] = sorted(layer_schema['properties'].keys()) + + r = client.post(rest_url, + data=data_xml, + headers=headers) + assert r.status_code == 200, f"{r.get_data()}" + + new_db_attributes = {} + new_wfs_properties = {} + for layername, attr_names in attr_names_list: + # test that exactly all attr_names were created in DB table + new_db_attributes[layername] = db.get_all_column_names(username, layername) + for attr_name in attr_names: + assert attr_name in new_db_attributes[layername], f"new_db_attributes={new_db_attributes[layername]}, attr_name={attr_name}" + assert set(attr_names).union(set(old_db_attributes[layername])) == set(new_db_attributes[layername]) + + # test that exactly all attr_names were distinguished also in WFS feature type + wfs = wfs_proxy(wfs_url) + layer_schema = wfs.get_schema(f"{username}:{layername}") + new_wfs_properties[layername] = sorted(layer_schema['properties'].keys()) + for attr_name in attr_names: + assert attr_name in new_wfs_properties[layername], f"new_wfs_properties={new_wfs_properties[layername]}, attr_name={attr_name}" + assert set(attr_names).union(set(old_wfs_properties[layername])) == set(new_wfs_properties[layername]),\ + set(new_wfs_properties[layername]).difference(set(attr_names).union(set(old_wfs_properties[layername]))) + + attr_names = ['inexisting_attribute_attr', 'inexisting_attribute_attr1a'] + data_xml = data_wfs.get_wfs20_insert_points_new_attr(username, layername, attr_names) + wfs_post(username, [(layername, attr_names)], data_xml) + + attr_names2 = ['inexisting_attribute_attr2'] + data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername, attr_names2) + wfs_post(username, [(layername, attr_names2)], data_xml) + + attr_names3 = ['inexisting_attribute_attr3'] + data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername, attr_names3, with_attr_namespace=True) + wfs_post(username, [(layername, attr_names3)], data_xml) + + attr_names4 = ['inexisting_attribute_attr4'] + data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername, attr_names4, with_filter=True) + wfs_post(username, [(layername, attr_names4)], data_xml) + + attr_names5 = ['inexisting_attribute_attr5'] + data_xml = data_wfs.get_wfs20_replace_points_new_attr(username, layername, attr_names5) + wfs_post(username, [(layername, attr_names5)], data_xml) + + attr_names_i1 = ['inexisting_attribute_attr_complex_i1'] + attr_names_i2 = ['inexisting_attribute_attr_complex_i2'] + attr_names_u = ['inexisting_attribute_attr_complex_u'] + attr_names_r = ['inexisting_attribute_attr_complex_r'] + attr_names_complex = [(layername, attr_names_i1 + attr_names_r), (layername2, attr_names_i2 + attr_names_u)] + data_xml = data_wfs.get_wfs20_complex_new_attr(username=username, + layername1=layername, + layername2=layername2, + attr_names_insert1=attr_names_i1, + attr_names_insert2=attr_names_i2, + attr_names_update=attr_names_u, + attr_names_replace=attr_names_r + ) + wfs_post(username, attr_names_complex, data_xml) + + attr_names6 = ['inexisting_attribute_attr6'] + data_xml = data_wfs.get_wfs10_insert_points_new_attr(username, layername, attr_names6) + wfs_post(username, [(layername, attr_names6)], data_xml) + + attr_names7 = ['inexisting_attribute_attr7'] + data_xml = data_wfs.get_wfs11_insert_points_new_attr(username, layername, attr_names7) + wfs_post(username, [(layername, attr_names7)], data_xml) + + attr_names8 = ['inexisting_attribute_attr8'] + data_xml = data_wfs.get_wfs10_update_points_new(username, layername, attr_names8, with_attr_namespace=True) + wfs_post(username, [(layername, attr_names8)], data_xml) + + attr_names9 = ['inexisting_attribute_attr9'] + data_xml = data_wfs.get_wfs10_update_points_new(username, layername, attr_names9, with_filter=True) + wfs_post(username, [(layername, attr_names9)], data_xml) + + attr_names10 = ['inexisting_attribute_attr10'] + data_xml = data_wfs.get_wfs11_insert_polygon_new_attr(username, layername, attr_names10) + wfs_post(username, [(layername, attr_names10)], data_xml) + + client_util.delete_layer(username, layername) + client_util.delete_layer(username, layername2) + + +def test_missing_attribute_authz(liferay_mock): + username = 'testmissingattr_authz' + layername1 = 'testmissingattr_authz_layer' + username2 = 'testmissingattr_authz2' + + authn_headers1 = get_auth_header(username) + authn_headers2 = get_auth_header(username2) + headers1 = { + 'Accept': 'text/xml', + 'Content-type': 'text/xml', + **authn_headers1, + } + headers2 = { + 'Accept': 'text/xml', + 'Content-type': 'text/xml', + **authn_headers2, + } + + def do_test(wfs_query, attribute_names): + # Test, that unauthorized user will not cause new attribute + with app.app_context(): + old_db_attributes = db.get_all_column_names(username, layername1) + for attr_name in attribute_names: + assert attr_name not in old_db_attributes, f"old_db_attributes={old_db_attributes}, attr_name={attr_name}" + r = requests.post(rest_url, + data=wfs_query, + headers=headers2) + assert r.status_code == 400, r.text + with app.app_context(): + new_db_attributes = db.get_all_column_names(username, layername1) + for attr_name in attribute_names: + assert attr_name not in new_db_attributes, f"new_db_attributes={new_db_attributes}, attr_name={attr_name}" + + # Test, that authorized user will cause new attribute + r = requests.post(rest_url, + data=wfs_query, + headers=headers1) + assert r.status_code == 200, r.text + with app.app_context(): + new_db_attributes = db.get_all_column_names(username, layername1) + for attr_name in attribute_names: + assert attr_name in new_db_attributes, f"new_db_attributes={new_db_attributes}, attr_name={attr_name}" + + layman_process = process.start_layman(dict({ + 'LAYMAN_AUTHZ_MODULE': 'layman.authz.read_everyone_write_owner', + }, **AUTHN_SETTINGS)) + + setup_user_layer(username, layername1, authn_headers1) + + rest_url = f"http://{settings.LAYMAN_SERVER_NAME}/geoserver/{username}/wfs?request=Transaction" + + # Testing, that user2 is not able to write to layer of user1 + + client_util.reserve_username(username2, headers=authn_headers2) + + # INSERT + attr_names = ['inexisting_attribute_auth1', 'inexisting_attribute_auth2'] + data_xml = data_wfs.get_wfs20_insert_points_new_attr(username, layername1, attr_names) + do_test(data_xml, attr_names) + + # UPDATE + attr_names = ['inexisting_attribute_auth3', 'inexisting_attribute_auth4'] + data_xml = data_wfs.get_wfs20_update_points_new_attr(username, layername1, attr_names) + do_test(data_xml, attr_names) + + client_util.delete_layer(username, layername1, headers1) + process.stop_process(layman_process) diff --git a/src/layman/layer/db/__init__.py b/src/layman/layer/db/__init__.py index e03e9c0d6..6c59e92c8 100644 --- a/src/layman/layer/db/__init__.py +++ b/src/layman/layer/db/__init__.py @@ -2,7 +2,7 @@ import math import os import psycopg2 -from flask import g +from flask import g, app from layman.common.language import get_languages_iso639_2 from layman.http import LaymanError @@ -167,6 +167,22 @@ def get_text_column_names(username, layername, conn_cur=None): return [r[0] for r in rows] +def get_all_column_names(username, layername, conn_cur=None): + conn, cur = conn_cur or get_connection_cursor() + + try: + cur.execute(f""" +SELECT QUOTE_IDENT(column_name) AS column_name +FROM information_schema.columns +WHERE table_schema = '{username}' +AND table_name = '{layername}' +""") + except: + raise LaymanError(7) + rows = cur.fetchall() + return [r[0] for r in rows] + + def get_number_of_features(username, layername, conn_cur=None): conn, cur = conn_cur or get_connection_cursor() @@ -464,3 +480,46 @@ def get_most_frequent_lower_distance2(username, layername, conn_cur=None): if freq / num_distances > 0.03: result = distance return result + + +def create_string_attributes(attribute_tuples, conn_cur=None): + conn, cur = conn_cur or get_connection_cursor() + query = "\n".join([f"""ALTER TABLE {username}.{layername} ADD COLUMN {attrname} VARCHAR(1024);""" for username, layername, attrname in attribute_tuples]) + "\n COMMIT;" + try: + cur.execute(query) + except: + raise LaymanError(7) + + +def get_missing_attributes(attribute_tuples, conn_cur=None): + conn, cur = conn_cur or get_connection_cursor() + + # Find all triples which do not already exist + query = f"""select attribs.* +from (""" + "\n union all\n".join([f"select '{username}' username, '{layername}' layername, '{attrname}' attrname" for username, layername, attrname in attribute_tuples]) + """) attribs left join + information_schema.columns c on c.table_schema = attribs.username + and c.table_name = attribs.layername + and c.column_name = attribs.attrname +where c.column_name is null""" + + try: + if attribute_tuples: + cur.execute(query) + except: + raise LaymanError(7) + + missing_attributes = set() + rows = cur.fetchall() + for row in rows: + missing_attributes.add((row[0], + row[1], + row[2])) + return missing_attributes + + +def ensure_attributes(attribute_tuples): + conn_cur = get_connection_cursor() + missing_attributes = get_missing_attributes(attribute_tuples, conn_cur) + if missing_attributes: + create_string_attributes(missing_attributes, conn_cur) + return missing_attributes diff --git a/src/layman/layer/rest_test.py b/src/layman/layer/rest_test.py index 4bd4b394e..b90db1a95 100644 --- a/src/layman/layer/rest_test.py +++ b/src/layman/layer/rest_test.py @@ -29,6 +29,7 @@ from .micka import csw from layman.common.micka import util as micka_common_util from layman.common.metadata import prop_equals_strict, PROPERTIES +from test.data import wfs as data_wfs from test import client as client_util @@ -1194,7 +1195,7 @@ def test_layer_with_different_geometry(client): 'Content-type': 'text/xml', } - data_xml = client_util.get_wfs_insert_points(username, layername) + data_xml = data_wfs.get_wfs20_insert_points(username, layername) r = requests.post(url_path_ows, data=data_xml, @@ -1210,7 +1211,7 @@ def test_layer_with_different_geometry(client): ) assert r.status_code == 200, f"HTTP Error {r.status_code}\n{r.text}" - data_xml2 = client_util.get_wfs_insert_lines(username, layername) + data_xml2 = data_wfs.get_wfs20_insert_lines(username, layername) r = requests.post(url_path_ows, data=data_xml2, diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index d3859e9cf..d7db99ea9 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -15,6 +15,7 @@ from layman.common import metadata as common_md LAYERNAME_RE = USERNAME_RE +ATTRNAME_RE = USERNAME_RE FLASK_PROVIDERS_KEY = f'{__name__}:PROVIDERS' FLASK_SOURCES_KEY = f'{__name__}:SOURCES' diff --git a/test/client.py b/test/client.py index 24b17145b..f2cb2bfa3 100644 --- a/test/client.py +++ b/test/client.py @@ -97,61 +97,3 @@ def reserve_username(username, headers=None): assert r.status_code == 200, r.text claimed_username = r.json()['username'] assert claimed_username == username - - -def get_wfs_insert_points(username, layername): - return f''' - - - <{username}:{layername}> - <{username}:wkb_geometry> - - 1.27108004304E7 2548415.5977 - - - - - ''' - - -def get_wfs_insert_lines(username, layername): - return f''' - - - <{username}:{layername}> - <{username}:wkb_geometry> - - - - 3722077.1689 5775850.1007 3751406.9331 5815606.0102 3830548.3984 5781176.5357 - 3866350.4899 5774848.8358 3880796.9478 5743277.797 3897591.3679 5738418.6547 - - - - - - - - ''' diff --git a/test/data/wfs.py b/test/data/wfs.py new file mode 100644 index 000000000..2cbeefc02 --- /dev/null +++ b/test/data/wfs.py @@ -0,0 +1,424 @@ +def get_wfs20_insert_points(username, layername): + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + + + ''' + + +def get_wfs20_insert_lines(username, layername): + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + + + 3722077.1689 5775850.1007 3751406.9331 5815606.0102 3830548.3984 5781176.5357 + 3866350.4899 5774848.8358 3880796.9478 5743277.797 3897591.3679 5738418.6547 + + + + + + + + ''' + + +def get_wfs20_insert_points_new_attr(username, layername, attr_names): + attr_xml = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names + ]) + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml} + + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.42108004308E7 2678415.5977 + + + <{username}:name>New name2 + <{username}:labelrank>4 + {attr_xml} + + +''' + + +def get_wfs10_insert_points_new_attr(username, layername, attr_names): + attr_xml = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names + ]) + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml} + + +''' + + +def get_wfs11_insert_points_new_attr(username, layername, attr_names): + attr_xml = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names + ]) + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + + + + 494475.71056415,5433016.8189323 494982.70115662,5435041.95096618 + + + + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml} + + +''' + + +def get_wfs11_insert_polygon_new_attr(username, layername, attr_names): + attr_xml = ' '.join([ + f"<{attr_name}>some value" + for attr_name in attr_names + ]) + return f''' + + + <{layername} xmlns="http://{username}"> + + + + + + 494475.71056415,5433016.8189323 494982.70115662,5435041.95096618 + + + + + + New name + 3 + {attr_xml} + + +''' + + +def get_wfs20_update_points_new_attr( + username, + layername, + attr_names, + with_attr_namespace=False, + with_filter=False, +): + attr_prefix = f"{username}:" if with_attr_namespace else '' + attr_xml = ' '.join([ + f""" + {attr_prefix}{attr_name} + some value + """ + for attr_name in attr_names + ]) + filter_xml = f""" + + + """ if with_filter else '' + return f''' + + + {attr_xml} + {filter_xml} + + ''' + + +def get_wfs10_update_points_new( + username, + layername, + attr_names, + with_attr_namespace=False, + with_filter=False, +): + attr_prefix = f"{username}:" if with_attr_namespace else '' + attr_xml = ' '.join([ + f""" + {attr_prefix}{attr_name} + some value + """ + for attr_name in attr_names + ]) + filter_xml = f""" + + + """ if with_filter else '' + return f''' + + + {attr_xml} + {filter_xml} + + ''' + + +def get_wfs20_replace_points_new_attr(username, layername, attr_names): + attr_xml = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names + ]) + return f''' + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml} + + + + + + + <{username}:{layername}> + <{username}:wkb_geometry> + + 1.42108004308E7 2678415.5977 + + + <{username}:name>New name2 + <{username}:labelrank>4 + {attr_xml} + + + + + +''' + + +def get_wfs20_complex_new_attr(username, + layername1, + layername2, + attr_names_insert1, + attr_names_insert2, + attr_names_update, + attr_names_replace): + with_attr_namespace = True + attr_xml_insert1 = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names_insert1 + ]) + attr_xml_insert2 = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names_insert2 + ]) + attr_prefix = f"{username}:" if with_attr_namespace else '' + attr_xml_update = ' '.join([ + f""" + {attr_prefix}{attr_name} + some value + """ + for attr_name in attr_names_update + ]) + attr_xml_replace = ' '.join([ + f"<{username}:{attr_name}>some value" + for attr_name in attr_names_replace + ]) + return f''' + + + <{username}:{layername1}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml_insert1} + + <{username}:{layername2}> + <{username}:wkb_geometry> + + 1.42108004308E7 2678415.5977 + + + <{username}:name>New name2 + <{username}:labelrank>4 + {attr_xml_insert2} + + + + {attr_xml_update} + + + <{username}:{layername1}> + <{username}:wkb_geometry> + + 1.27108004304E7 2548415.5977 + + + <{username}:name>New name + <{username}:labelrank>3 + {attr_xml_replace} + + + + + +''' diff --git a/test/process.py b/test/process.py index b6a50158d..f30cbe304 100644 --- a/test/process.py +++ b/test/process.py @@ -39,6 +39,8 @@ def liferay_mock(): 'test_rewe_rewo2': None, 'testproxy': None, 'testproxy2': None, + 'testmissingattr_authz': None, + 'testmissingattr_authz2': None, }, }, 'host': '0.0.0.0',