Test rewrite improvements

.github/workflows/code_quality.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
-      - run: npm install --global remark-cli remark-validate-links remark-lint-no-dead-urls
+      - run: npm install --global remark-cli@9 remark-validate-links remark-lint-no-dead-urls
       - run: make remark
 
   Flake_8_syntax_errors:

Makefile

@@ -214,7 +214,7 @@ test-bash:
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm layman_test bash
 
 lint:
-	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "pylint -f colorized -r y --disable=duplicate-except --disable=duplicate-code --disable=no-name-in-module --disable=too-many-lines --disable=line-too-long --disable=redefined-outer-name --disable=c-extension-no-member --disable=import-outside-toplevel --disable=too-many-locals --disable=redefined-builtin --disable=too-many-arguments --disable=wrong-import-position --disable=too-many-statements --disable=too-many-branches --max-line-length=127 ./src ./test_tools ./tests && flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ./src ./test_tools ./tests && pycodestyle --count --statistics --ignore=E402,E501,W503 ./src ./test_tools ./tests"
+	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "pylint -f colorized -j 2 -r y --disable=duplicate-except --disable=duplicate-code --disable=no-name-in-module --disable=too-many-lines --disable=line-too-long --disable=redefined-outer-name --disable=c-extension-no-member --disable=import-outside-toplevel --disable=too-many-locals --disable=redefined-builtin --disable=too-many-arguments --disable=wrong-import-position --disable=too-many-statements --disable=too-many-branches --max-line-length=127 ./src ./test_tools ./tests && flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ./src ./test_tools ./tests && pycodestyle --count --statistics --ignore=E402,E501,W503 ./src ./test_tools ./tests"
 
 lint-verbose:
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "pylint -f colorized -r y ./src ./test_tools ./tests"

src/layman/layer/rest_workspace_test.py

@@ -5,7 +5,6 @@
 import time
 import xml.etree.ElementTree as ET
 from urllib.parse import urljoin
-import difflib
 import logging
 import sys
 import requests
@@ -22,15 +21,13 @@
 from layman.layer import db
 from layman.layer.geoserver import wms as geoserver_wms, sld as geoserver_sld
 from layman import celery as celery_util
-from layman.common.micka import util as micka_common_util
 from layman.common.metadata import prop_equals_strict, PROPERTIES
 from layman.util import SimpleCounter
 from test_tools import flask_client, process_client
 from test_tools.data import wfs as data_wfs
 from test_tools.util import url_for, url_for_external
 from . import util, LAYER_TYPE
 from .geoserver.util import wms_proxy
-from .micka import csw
 
 logger = logging.getLogger(__name__)
 
@@ -464,35 +461,6 @@ def test_post_layers_shp(client):
         f'{LAYER_TYPE}': publication_counter.get()
     })
 
-    # assert metadata file is the same as filled template except for UUID
-    template_path, prop_values = csw.get_template_path_and_values(username, layername, http_method='post')
-    xml_file_object = micka_common_util.fill_xml_template_as_pretty_file_object(template_path, prop_values,
-                                                                                csw.METADATA_PROPERTIES)
-    expected_path = 'src/layman/layer/rest_test_filled_template.xml'
-    with open(expected_path) as file:
-        expected_lines = file.readlines()
-    diff_lines = list(difflib.unified_diff([line.decode('utf-8') for line in xml_file_object.readlines()], expected_lines))
-    plus_lines = [line for line in diff_lines if line.startswith('+ ')]
-    assert len(plus_lines) == 3, ''.join(diff_lines)
-    minus_lines = [line for line in diff_lines if line.startswith('- ')]
-    assert len(minus_lines) == 3, ''.join(diff_lines)
-    plus_line = plus_lines[0]
-    assert plus_line == '+    <gco:CharacterString>m-81c0debe-b2ea-4829-9b16-581083b29907</gco:CharacterString>\n', ''.join(
-        diff_lines)
-    minus_line = minus_lines[0]
-    assert minus_line.startswith('-    <gco:CharacterString>m') and minus_line.endswith(
-        '</gco:CharacterString>\n'), ''.join(diff_lines)
-    plus_line = plus_lines[1]
-    assert plus_line == '+    <gco:Date>2007-05-25</gco:Date>\n', ''.join(diff_lines)
-    minus_line = minus_lines[1]
-    assert minus_line.startswith('-    <gco:Date>') and minus_line.endswith('</gco:Date>\n'), ''.join(diff_lines)
-    plus_line = plus_lines[2]
-    assert plus_line == '+                <gco:Date>2019-12-07</gco:Date>\n', ''.join(diff_lines)
-    minus_line = minus_lines[2]
-    assert minus_line.startswith('-                <gco:Date>') and minus_line.endswith('</gco:Date>\n'), ''.join(
-        diff_lines)
-    assert len(diff_lines) == 29, ''.join(diff_lines)
-
 
 @pytest.mark.usefixtures('app_context', 'ensure_layman')
 def test_post_layers_layer_exists(client):

src/layman/layer/rest_test_filled_template.xml → test_tools/data/micka/rest_test_filled_template.xml

@@ -78,7 +78,7 @@
       <gmd:citation>
         <gmd:CI_Citation>
           <gmd:title>
-            <gco:CharacterString>ne_110m_admin_0_countries_shp</gco:CharacterString>
+            <gco:CharacterString>test_publications_same_name_publ</gco:CharacterString>
           </gmd:title>
           <gmd:date>
             <gmd:CI_Date>
@@ -93,7 +93,7 @@
           <gmd:identifier>
             <gmd:MD_Identifier>
               <gmd:code>
-                <gmx:Anchor xlink:href="http://enjoychallenge.tech/rest/workspaces/testuser1/layers/ne_110m_admin_0_countries_shp">ne_110m_admin_0_countries_shp</gmx:Anchor>
+                <gmx:Anchor xlink:href="http://enjoychallenge.tech/rest/workspaces/test_workspace_1/layers/test_publications_same_name_publ">test_publications_same_name_publ</gmx:Anchor>
               </gmd:code>
             </gmd:MD_Identifier>
           </gmd:identifier>
@@ -125,7 +125,7 @@
       <gmd:graphicOverview>
         <gmd:MD_BrowseGraphic>
           <gmd:fileName>
-            <gco:CharacterString>http://enjoychallenge.tech/rest/workspaces/testuser1/layers/ne_110m_admin_0_countries_shp/thumbnail</gco:CharacterString>
+            <gco:CharacterString>http://enjoychallenge.tech/rest/workspaces/test_workspace_1/layers/test_publications_same_name_publ/thumbnail</gco:CharacterString>
           </gmd:fileName>
           <gmd:fileType>
             <gco:CharacterString>PNG</gco:CharacterString>
@@ -269,7 +269,7 @@
           <gmd:onLine>
             <gmd:CI_OnlineResource>
               <gmd:linkage>
-                <gmd:URL>http://localhost:8000/geoserver/testuser1_wms/ows?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0&amp;LAYERS=ne_110m_admin_0_countries_shp</gmd:URL>
+                <gmd:URL>http://localhost:8000/geoserver/test_workspace_1_wms/ows?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0&amp;LAYERS=test_publications_same_name_publ</gmd:URL>
               </gmd:linkage>
               <gmd:protocol>
                 <gmx:Anchor xlink:href="https://services.cuzk.cz/registry/codelist/OnlineResourceProtocolValue/OGC:WMS-1.3.0-http-get-capabilities">OGC:WMS-1.3.0-http-get-capabilities</gmx:Anchor>
@@ -282,7 +282,7 @@
           <gmd:onLine>
             <gmd:CI_OnlineResource>
               <gmd:linkage>
-                <gmd:URL>http://localhost:8000/geoserver/testuser1/wfs?SERVICE=WFS&amp;REQUEST=GetCapabilities&amp;VERSION=2.0.0&amp;LAYERS=ne_110m_admin_0_countries_shp</gmd:URL>
+                <gmd:URL>http://localhost:8000/geoserver/test_workspace_1/wfs?SERVICE=WFS&amp;REQUEST=GetCapabilities&amp;VERSION=2.0.0&amp;LAYERS=test_publications_same_name_publ</gmd:URL>
               </gmd:linkage>
               <gmd:protocol>
                 <gmx:Anchor xlink:href="https://services.cuzk.cz/registry/codelist/OnlineResourceProtocolValue/OGC:WFS-2.0.0-http-get-capabilities">OGC:WFS-2.0.0-http-get-capabilities</gmx:Anchor>
@@ -295,7 +295,7 @@
           <gmd:onLine>
             <gmd:CI_OnlineResource>
               <gmd:linkage>
-                <gmd:URL>http://enjoychallenge.tech/rest/workspaces/testuser1/layers/ne_110m_admin_0_countries_shp</gmd:URL>
+                <gmd:URL>http://enjoychallenge.tech/rest/workspaces/test_workspace_1/layers/test_publications_same_name_publ</gmd:URL>
               </gmd:linkage>
               <gmd:protocol>
                 <gmx:Anchor xlink:href="https://services.cuzk.cz/registry/codelist/OnlineResourceProtocolValue/WWW:LINK-1.0-http--link">WWW:LINK-1.0-http--link</gmx:Anchor>

tests/static_publications/__init__.py

@@ -163,7 +163,10 @@
     },
     (COMMON_WORKSPACE, LAYER_TYPE, 'post_jp2'): {
         DEFINITION: [
-            {'file_paths': ['sample/layman.layer/sample_jp2_rgb.jp2', ]},
+            {'file_paths': ['sample/layman.layer/sample_jp2_rgb.jp2', ],
+             'headers': HEADERS[OWNER],
+             'access_rights': {'read': OWNER, 'write': OWNER},
+             },
         ],
         TEST_DATA: {
             'bbox': (1829708, 6308828.600, 1833166.200, 6310848.600),
@@ -172,6 +175,8 @@
             'thumbnail': '/code/test_tools/data/thumbnail/raster_layer_jp2.png',
             'file_type': settings.FILE_TYPE_RASTER,
             'style_type': 'sld',
+            'users_can_read': [OWNER],
+            'users_can_write': [OWNER],
         },
     },
     (COMMON_WORKSPACE, LAYER_TYPE, 'patch_jp2'): {
@@ -351,12 +356,37 @@
     },
     (WORKSPACE1, LAYER_TYPE, 'test_publications_same_name_publ'): {
         DEFINITION: [
-            {},
+            {'file_paths': [
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.cpg',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.dbf',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.prj',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.README.html',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shp',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.shx',
+                'tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.VERSION.txt',
+            ]},
         ],
         TEST_DATA: {
-            'bbox': (1571204.369948366, 6268896.225570714, 1572590.854206196, 6269876.33561699),
+            'bbox': (-20026376.39, -20048966.1, 20026376.39, 18440002.895114236),
             'file_type': settings.FILE_TYPE_VECTOR,
             'style_type': 'sld',
+            'micka_xml': {'filled_template': 'test_tools/data/micka/rest_test_filled_template.xml',
+                          'diff_lines': [
+                              {'plus_line': '+    <gco:CharacterString>m-81c0debe-b2ea-4829-9b16-581083b29907</gco:CharacterString>\n',
+                               'minus_line_starts_with': '-    <gco:CharacterString>m',
+                               'minus_line_ends_with': '</gco:CharacterString>\n',
+                               },
+                              {'plus_line': '+    <gco:Date>2007-05-25</gco:Date>\n',
+                               'minus_line_starts_with': '-    <gco:Date>',
+                               'minus_line_ends_with': '</gco:Date>\n',
+                               },
+                              {'plus_line': '+                <gco:Date>2019-12-07</gco:Date>\n',
+                               'minus_line_starts_with': '-                <gco:Date>',
+                               'minus_line_ends_with': '</gco:Date>\n',
+                               },
+                          ],
+                          'diff_lines_len': 29,
+                          },
         },
     },
     (WORKSPACE2, LAYER_TYPE, 'test_publications_same_name_publ'): {

tests/static_publications/data.py

@@ -47,7 +47,7 @@ def ensure_test_data(liferay_mock):
         info = util.get_publication_infos()
 
     for workspace, publ_type, publication in info:
-        headers = data.HEADERS.get(workspace)
+        headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
         process_client.delete_workspace_publication(publ_type, workspace, publication, headers=headers)
         assert_publication_after_delete(workspace, publ_type, publication)
 

tests/static_publications/single_publication/layers_test.py

@@ -1,5 +1,6 @@
 import os
 from urllib.parse import urljoin
+import difflib
 import requests
 from lxml import etree as ET
 from owslib.wms import WebMapService
@@ -8,8 +9,10 @@
 from geoserver import GS_REST_WORKSPACES, GS_REST, GS_AUTH, util as gs_util
 from layman import settings, app, util as layman_util
 from layman.common import bbox as bbox_util, geoserver as gs_common
+from layman.common.micka import util as micka_common_util
 from layman.layer import util as layer_util, db as layer_db
 from layman.layer.geoserver.wms import DEFAULT_WMS_QGIS_STORE_PREFIX, VERSION
+from layman.layer.micka import csw
 from layman.layer.qgis import util as qgis_util
 from test_tools import process_client, assert_util, geoserver_client
 from test_tools.util import url_for
@@ -30,7 +33,7 @@ def test_info(workspace, publ_type, publication):
     ensure_publication(workspace, publ_type, publication)
     wms_url = f"http://localhost:8000/geoserver/{workspace}{settings.LAYMAN_GS_WMS_WORKSPACE_POSTFIX}/ows"
     wfs_url = f"http://localhost:8000/geoserver/{workspace}/wfs"
-    headers = data.HEADERS.get(workspace)
+    headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
     style = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA]['style_type']
 
     info = process_client.get_workspace_publication(publ_type, workspace, publication, headers=headers)
@@ -88,7 +91,7 @@ def test_geoserver_workspace(workspace, publ_type, publication):
 @pytest.mark.usefixtures('liferay_mock', 'ensure_layman')
 def test_get_layer_style(workspace, publ_type, publication):
     ensure_publication(workspace, publ_type, publication)
-    headers = data.HEADERS.get(workspace)
+    headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
 
     with app.app_context():
         rest_url = url_for('rest_workspace_layer_style.get', workspace=workspace, layername=publication)
@@ -166,20 +169,18 @@ def test_wms_layer(workspace, publ_type, publication):
 
         assert_util.assert_same_images(url, obtained_file, expected_file, pixel_tolerance)
 
-    file_type = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('file_type')
-    if file_type != settings.FILE_TYPE_RASTER:
-        authn_headers = data.HEADERS.get(workspace)
-        for service_endpoint in {'ows', 'wms'}:
-            wms_url = geoserver_client.get_wms_url(workspace, service_endpoint)
+    gs_workspace = info['_wms']['workspace']
+    authn_headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
+    for service_endpoint in {'ows', 'wms'}:
+        wms_url = geoserver_client.get_wms_url(gs_workspace, service_endpoint)
 
-            layer_info = process_client.get_workspace_layer(workspace, publication, headers=authn_headers)
-            tn_bbox = gs_util.get_square_bbox(layer_info['bounding_box'])
+        layer_info = process_client.get_workspace_layer(workspace, publication, headers=authn_headers)
+        tn_bbox = gs_util.get_square_bbox(layer_info['bounding_box'])
 
-            response = gs_util.get_layer_thumbnail(wms_url, publication, tn_bbox, headers=authn_headers, wms_version=VERSION)
-            response.raise_for_status()
-            assert 'image' in response.headers['content-type']
+        response = gs_util.get_layer_thumbnail(wms_url, publication, tn_bbox, headers=authn_headers, wms_version=VERSION)
+        response.raise_for_status()
+        assert 'image' in response.headers['content-type'], f'response.headers={response.headers}, response.content={response.content}'
 
-    gs_workspace = info['_wms']['workspace']
     all_auth_info = util.get_users_and_headers_for_publication(workspace, publ_type, publication)
     headers_list_in = all_auth_info['read'][util.KEY_AUTH][util.KEY_HEADERS]
     headers_list_out = all_auth_info['read'][util.KEY_NOT_AUTH][util.KEY_HEADERS]
@@ -271,3 +272,35 @@ def test_gs_data_security(workspace, publ_type, publication):
             assert gs_expected_roles == gs_roles\
                 or (is_personal_workspace
                     and gs_expected_roles == owner_and_everyone_roles == gs_roles.union(owner_role_set)), f'gs_expected_roles={gs_expected_roles}, gs_roles={gs_roles}, wspace={wspace}, is_personal_workspace={is_personal_workspace}'
+
+
+@pytest.mark.parametrize('workspace, publ_type, publication', [(wspace, ptype, pub)
+                                                               for wspace, ptype, pub in data.LIST_LAYERS
+                                                               if data.PUBLICATIONS[(wspace, ptype, pub)][data.TEST_DATA].get('micka_xml')])
+@pytest.mark.usefixtures('liferay_mock', 'ensure_layman')
+def test_micka_xml(workspace, publ_type, publication):
+    ensure_publication(workspace, publ_type, publication)
+
+    # assert metadata file is the same as filled template except for UUID
+    with app.app_context():
+        template_path, prop_values = csw.get_template_path_and_values(workspace, publication, http_method='post')
+    xml_file_object = micka_common_util.fill_xml_template_as_pretty_file_object(template_path, prop_values,
+                                                                                csw.METADATA_PROPERTIES)
+    micka_xml_def = data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA]['micka_xml']
+    expected_path = micka_xml_def['filled_template']
+    with open(expected_path) as file:
+        expected_lines = file.readlines()
+    exp_diff_lines = micka_xml_def['diff_lines']
+    diff_lines = list(difflib.unified_diff([line.decode('utf-8') for line in xml_file_object.readlines()], expected_lines))
+    plus_lines = [line for line in diff_lines if line.startswith('+ ')]
+    assert len(plus_lines) == len(exp_diff_lines), ''.join(diff_lines)
+    minus_lines = [line for line in diff_lines if line.startswith('- ')]
+    assert len(minus_lines) == len(exp_diff_lines), ''.join(diff_lines)
+    for idx, diff_line in enumerate(exp_diff_lines):
+        plus_line = plus_lines[idx]
+        minus_line = minus_lines[idx]
+        assert plus_line == diff_line['plus_line'], f'diff_lines={"".join(diff_lines)}, idx={idx}, diff_line={diff_line}'
+        assert minus_line.startswith(diff_line['minus_line_starts_with']), f'diff_lines={"".join(diff_lines)}, idx={idx}, diff_line={diff_line}'
+        assert minus_line.endswith(diff_line['minus_line_ends_with']), f'diff_lines={"".join(diff_lines)}, idx={idx}, diff_line={diff_line}'
+
+    assert len(diff_lines) == micka_xml_def['diff_lines_len'], ''.join(diff_lines)

tests/static_publications/single_publication/patch_error_test.py

@@ -17,8 +17,9 @@ def test_patch_raster_qml(workspace, publ_type, publication):
                     'detail': 'Raster layers are not allowed to have QML style.',
                     }
 
+    headers = data.HEADERS.get(data.PUBLICATIONS[(workspace, publ_type, publication)][data.TEST_DATA].get('users_can_write', [None])[0])
     with pytest.raises(LaymanError) as exc_info:
-        process_client.patch_workspace_layer(workspace, publication, style_file='sample/style/ne_10m_admin_0_countries.qml')
+        process_client.patch_workspace_layer(workspace, publication, style_file='sample/style/ne_10m_admin_0_countries.qml', headers=headers)
     test_util.assert_error(expected_exc, exc_info)