Automatically create not-yet-existing attribute on WFS-T (insert, update)

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Changes
 - [#65](https://github.com/jirik/layman/issues/65) [WFS endpoint](doc/rest.md#get-layer) accepts same [authentication](doc/security.md#authentication) credentials (e.g. [OAuth2 headers](doc/oauth2/index.md#request-layman-rest-api)) as Layman REST API endpoints. It's implemented using Layman's WFS proxy. This proxy authenticates the user and send user identification to GeoServer. In combination with changes in v1.6.0, Layman's [`read-everyone-write-owner` authorization](doc/security.md#authorization) (when active) is propagated to GeoServer and user can change only hers layers.
 - [#65](https://github.com/jirik/layman/issues/65) Layman automatically setup [HTTP authentication attribute](https://docs.geoserver.org/stable/en/user/security/tutorials/httpheaderproxy/index.html) and chain filter at startup. Secret value of this attribute can be changed in [LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE](doc/env-settings.md#LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE) and is used by Layman's WFS proxy.
+- [#95](https://github.com/jirik/layman/issues/95) When calling WFS, Layman will automatically create missing attributes in DB before redirecting request to GeoServer. If creating attribute fails for any reason, warning in log is created and request is redirecting nevertheless.
 
 ## v1.6.1
 2020-08-19

src/layman/authz/read_everyone_write_everyone.py

@@ -22,3 +22,7 @@ def get_gs_roles(username, type):
     elif type == 'w':
         roles = gs.get_roles_anyone(username)
     return roles
+
+
+def can_i_edit(publ_type, username, publication_name):
+    return True

src/layman/authz/read_everyone_write_owner.py

@@ -43,3 +43,7 @@ def get_gs_roles(username, type):
     elif type == 'w':
         roles = gs.get_roles_owner(username)
     return roles
+
+
+def can_i_edit(publ_type, username, publication_name):
+    return g.user is not None and g.user['username'] == username

src/layman/common/geoserver/__init__.py

@@ -467,3 +467,13 @@ def get_roles_anyone(username):
 def get_roles_owner(username):
     roles = {username_to_rolename(username)}
     return roles
+
+
+def reset(auth):
+    logger.info(f"Resetting GeoServer")
+    r_url = settings.LAYMAN_GS_REST + 'reset'
+    r = requests.post(r_url,
+                      headers=headers_json,
+                      auth=auth
+                      )
+    r.raise_for_status()

src/layman/gs_wfs_proxy.py

@@ -1,9 +1,20 @@
+import re
+import traceback
+
 import requests
+from lxml import etree as ET
 
 from flask import Blueprint, g, current_app as app, request, Response
 
 from layman.authn import authenticate
 from layman import settings
+from layman.layer import db
+from layman.layer.util import LAYERNAME_RE, ATTRNAME_RE
+from layman.util import USERNAME_ONLY_PATTERN
+from layman.common.geoserver import reset as gs_reset
+from layman.layer import LAYER_TYPE
+from layman.authz import util as authz
+
 
 bp = Blueprint('gs_wfs_proxy_bp', __name__)
 
@@ -14,19 +25,135 @@ def before_request():
     pass
 
 
+def ensure_wfs_t_attributes(binary_data):
+    try:
+        xml_tree = ET.XML(binary_data)
+        version = xml_tree.get('version')[0:4]
+        if version not in ["2.0.", "1.0.", "1.1."] or xml_tree.get('service').upper() != "WFS":
+            app.logger.warning(f"WFS Proxy: only xml version 2.0, 1.1, 1.0 and WFS service are supported. Request "
+                               f"only redirected. Version={xml_tree.get('version')}, service={xml_tree.get('service')}")
+            return
+
+        authz_module = authz.get_authz_module()
+        attribs = set()
+        for action in xml_tree:
+            action_qname = ET.QName(action)
+            if action_qname.localname in ('Insert', 'Replace'):
+                extracted_attribs = extract_attributes_from_wfs_t_insert_replace(action, authz_module)
+                attribs.update(extracted_attribs)
+            elif action_qname.localname in ('Update'):
+                extracted_attribs = extract_attributes_from_wfs_t_update(action,
+                                                                         authz_module,
+                                                                         xml_tree,
+                                                                         major_version=version[0:1])
+                attribs.update(extracted_attribs)
+
+        app.logger.info(f"GET WFS check_xml_for_attribute attribs={attribs}")
+        if attribs:
+            created_attributes = db.ensure_attributes(attribs)
+            if created_attributes:
+                gs_reset(settings.LAYMAN_GS_AUTH)
+
+    except BaseException as err:
+        app.logger.warning(f"WFS Proxy: error={err}, trace={traceback.format_exc()}")
+
+
+def extract_attributes_from_wfs_t_update(action, authz_module, xml_tree, major_version="2"):
+    attribs = set()
+    layer_qname = action.get('typeName').split(':')
+    ws_namespace = layer_qname[0]
+    ws_match = re.match(r"^(" + USERNAME_ONLY_PATTERN + ")$", ws_namespace)
+    if ws_match:
+        ws_name = ws_match.group(1)
+    else:
+        app.logger.warning(f"WFS Proxy: skipping due to wrong namespace name. Namespace={ws_namespace}")
+        return attribs
+    layer_name = layer_qname[1]
+    layer_match = re.match(LAYERNAME_RE, layer_name)
+    if not layer_match:
+        app.logger.warning(f"WFS Proxy: skipping due to wrong layer name. Layer name={layer_name}")
+        return attribs
+    if not authz_module.can_i_edit(LAYER_TYPE, ws_name, layer_name):
+        app.logger.warning(f"Can not edit. ws_namespace={ws_namespace}")
+        return attribs
+    print(f"Major version =!{major_version}!")
+    value_ref_string = "Name" if major_version == "1" else "ValueReference"
+    print(f"value_ref_string ={value_ref_string}")
+    properties = action.xpath('wfs:Property/wfs:' + value_ref_string, namespaces=xml_tree.nsmap)
+    for prop in properties:
+        split_text = prop.text.split(':')
+        # No namespace in element text
+        if len(split_text) == 1:
+            attrib_name = split_text[0]
+        # There is namespace in element text
+        elif len(split_text) == 2:
+            if split_text[0] != ws_namespace:
+                continue
+            attrib_name = split_text[1]
+        attrib_match = re.match(ATTRNAME_RE, attrib_name)
+        if not attrib_match:
+            app.logger.warning(f"WFS Proxy: skipping due to wrong attribute name."
+                               f"Attribute name={attrib_name}")
+            continue
+        attribs.add((ws_name,
+                     layer_name,
+                     attrib_name))
+    return attribs
+
+
+def extract_attributes_from_wfs_t_insert_replace(action, authz_module):
+    attribs = set()
+    for layer in action:
+        layer_qname = ET.QName(layer)
+        ws_namespace = layer_qname.namespace
+        ws_match = re.match(r"^http://(" + USERNAME_ONLY_PATTERN + ")$", ws_namespace)
+        if ws_match:
+            ws_name = ws_match.group(1)
+        else:
+            app.logger.warning(f"WFS Proxy: skipping due to wrong namespace name. Namespace={ws_namespace}")
+            continue
+        layer_name = layer_qname.localname
+        layer_match = re.match(LAYERNAME_RE, layer_name)
+        if not layer_match:
+            app.logger.warning(f"WFS Proxy: skipping due to wrong layer name. Layer name={layer_name}")
+            continue
+        if not authz_module.can_i_edit(LAYER_TYPE, ws_name, layer_name):
+            app.logger.warning(f"WFS Proxy: Can not edit. ws_namespace={ws_name}")
+            continue
+        for attrib in layer:
+            attrib_qname = ET.QName(attrib)
+            if attrib_qname.namespace != ws_namespace:
+                app.logger.warning(f"WFS Proxy: skipping due to different namespace in layer and in "
+                                   f"attribute. Layer namespace={ws_namespace}, a"
+                                   f"ttribute namespace={attrib_qname.namespace}")
+                continue
+            attrib_name = attrib_qname.localname
+            attrib_match = re.match(ATTRNAME_RE, attrib_name)
+            if not attrib_match:
+                app.logger.warning(f"WFS Proxy: skipping due to wrong attribute name. Attribute name={attrib_name}")
+                continue
+            attribs.add((ws_name,
+                         layer_name,
+                         attrib_name))
+    return attribs
+
+
 @bp.route('/<path:subpath>', methods=['POST', 'GET'])
 def proxy(subpath):
     app.logger.info(f"GET WFS proxy, user={g.user}, subpath={subpath}, url={request.url}, request.query_string={request.query_string.decode('UTF-8')}")
 
     url = settings.LAYMAN_GS_URL + subpath + '?' + request.query_string.decode('UTF-8')
     headers_req = {key.lower(): value for (key, value) in request.headers if key.lower() not in ['host', settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE.lower()]}
+    data = request.get_data()
     if g.user is not None:
         headers_req[settings.LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE] = g.user['username']
 
     app.logger.info(f"GET WFS proxy, headers_req={headers_req}, url={url}")
+    if data is not None and len(data) > 0:
+        ensure_wfs_t_attributes(data)
     response = requests.request(method=request.method,
                                 url=url,
-                                data=request.get_data(),
+                                data=data,
                                 headers=headers_req,
                                 cookies=request.cookies,
                                 allow_redirects=False