diff --git a/.env.demo b/.env.demo index a6c741df0..c60c6a262 100644 --- a/.env.demo +++ b/.env.demo @@ -60,7 +60,7 @@ LAYMAN_CLIENT_URL=http://layman_client:3000/client/ # client LAYMAN_CLIENT_PUBLIC_URL=http://localhost/client/ -LAYMAN_CLIENT_VERSION=ab35c7cca3ca5f8b1323405b9e7a17a099cbc214 +LAYMAN_CLIENT_VERSION=1af9250d1451edc3142febcc6ae70ad9c4728970 # extra hosts to be added to /etc/hosts EXTRA_HOST1=1.2.3.4:1.2.3.4 diff --git a/.env.dev b/.env.dev index 8e6272d72..b29540b2d 100644 --- a/.env.dev +++ b/.env.dev @@ -62,7 +62,7 @@ LAYMAN_CLIENT_URL=http://layman_client:3000/client/ # client LAYMAN_CLIENT_PUBLIC_URL=http://localhost:3000/client/ -LAYMAN_CLIENT_VERSION=ab35c7cca3ca5f8b1323405b9e7a17a099cbc214 +LAYMAN_CLIENT_VERSION=1af9250d1451edc3142febcc6ae70ad9c4728970 ############################################################################## diff --git a/.env.test b/.env.test index 9f224de38..c79d435f3 100644 --- a/.env.test +++ b/.env.test @@ -62,7 +62,7 @@ LAYMAN_CLIENT_URL=http://layman_client_test:3000/client/ # client LAYMAN_CLIENT_PUBLIC_URL=http://layman_test_run_1:8000/client/ -LAYMAN_CLIENT_VERSION=ab35c7cca3ca5f8b1323405b9e7a17a099cbc214 +LAYMAN_CLIENT_VERSION=1af9250d1451edc3142febcc6ae70ad9c4728970 ############################################################################## diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8f8c8ff..a21099d11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ ### Migrations and checks - [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for full-text filtering (install [unaccent](https://www.postgresql.org/docs/10/unaccent.html), create immutable `my_unaccent` function, index unaccented `title` column in `publications` table). +- [#257](https://github.com/jirik/layman/issues/257) Adjust prime DB schema for ordering by last change (create and fill column `updated_at` in `publications` table). ### Changes -- [#257](https://github.com/jirik/layman/issues/257) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Maps](doc/rest.md#get-maps) can filter and reorder results according to new query parameters. +- [#257](https://github.com/jirik/layman/issues/257) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Maps](doc/rest.md#get-maps) can filter and reorder results according to new query parameters. +- [#257](https://github.com/jirik/layman/issues/257) Responses of [GET Layers](doc/rest.md#get-layers), [GET Workspace Layers](doc/rest.md#get-workspace-layer), [GET Workspace Layer](doc/rest.md#get-workspace-layer), [PATCH Workspace Layer](doc/rest.md#patch-workspace-layer), [GET Maps](doc/rest.md#get-maps), [GET Workspace Maps](doc/rest.md#get-workspace-map), [GET Workspace Map](doc/rest.md#get-workspace-map), and [PATCH Workspace Map](doc/rest.md#patch-workspace-map) contains new attribute `updated_at` with date and time of last PATCH/POST request to given publication. ## v1.11.0 2021-03-16 diff --git a/doc/rest.md b/doc/rest.md index 52347ae77..c952121f1 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -39,6 +39,7 @@ Query parameters: - *order_by*: String. Can be one of these values: - `full_text` Publications will be ordered by results of full-text search. Can be used only in combination with *full_text_filter*. - `title` Publications will be ordered lexicographically by title value. + - `last_change` Publications will be ordered by time of last change. Recently updated publications will be first. #### Response Content-Type: `application/json` @@ -49,6 +50,7 @@ JSON array of objects representing available layers with following structure: - **title**: String. Title of the layer. - **uuid**: String. UUID of the layer. - **url**: String. URL of the layer. It points to [GET Workspace Layer](#get-workspace-layer). +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **access_rights**: - **read**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [read access](./security.md#Authorization). - **write**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [write access](./security.md#Authorization). @@ -71,6 +73,7 @@ JSON array of objects representing available layers with following structure: - **title**: String. Title of the layer. - **uuid**: String. UUID of the layer. - **url**: String. URL of the layer. It points to [GET Workspace Layer](#get-workspace-layer). +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **access_rights**: - **read**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [read access](./security.md#Authorization). - **write**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [write access](./security.md#Authorization). @@ -195,6 +198,7 @@ JSON object with following structure: - **url**: String. URL pointing to this endpoint. - **title**: String. - **description**: String. +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **wms** - *url*: String. URL of WMS endpoint. It points to WMS endpoint of appropriate GeoServer workspace. - *status*: Status information about GeoServer import and availability of WMS layer. No status object means the source is available. Usual state values are @@ -400,6 +404,7 @@ Query parameters: - *order_by*: String. Can be one of these values: - `full_text` Publications will be ordered by results of full-text search. Can be used only in combination with *full_text_filter*. - `title` Publications will be ordered lexicographically by title value. + - `last_change` Publications will be ordered by time of last change. Recently updated publications will be first. #### Response Content-Type: `application/json` @@ -410,6 +415,7 @@ JSON array of objects representing available maps with following structure: - **title**: String. Title of the map. - **uuid**: String. UUID of the map. - **url**: String. URL of the map. It points to [GET Workspace Map](#get-workspace-map). +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **access_rights**: - **read**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [read access](./security.md#Authorization). - **write**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [write access](./security.md#Authorization). @@ -433,6 +439,7 @@ JSON array of objects representing available maps with following structure: - **title**: String. Title of the map. - **uuid**: String. UUID of the map. - **url**: String. URL of the map. It points to [GET Workspace Map](#get-workspace-map). +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **access_rights**: - **read**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [read access](./security.md#Authorization). - **write**: Array of strings. Names of [users](./models.md#user) and [roles](./models.md#role) with [write access](./security.md#Authorization). @@ -525,6 +532,7 @@ JSON object with following structure: - **url**: String. URL pointing to this endpoint. - **title**: String. Taken from `title` attribute of JSON root object - **description**: String. Taken from `abstract` attribute of JSON root object. +- **updated_at**: String. Date and time of last POST/PATCH of the publication. Format is [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601), more specifically `YYYY-MM-DDThh:mm:ss.sss±hh:mm`, always in UTC. Sample value: `"2021-03-18T09:29:53.769233+00:00"` - **file** - *url*: String. URL of map-composition JSON file. It points to [GET Workspace Map File](#get-workspace-map-file). - *path*: String. Path to map-composition JSON file, relative to workspace directory. diff --git a/src/layman/common/get_publications_consts.py b/src/layman/common/get_publications_consts.py index 775aabdcd..cf1c11fab 100644 --- a/src/layman/common/get_publications_consts.py +++ b/src/layman/common/get_publications_consts.py @@ -4,3 +4,4 @@ ORDER_BY_FULL_TEXT = 'full_text' ORDER_BY_TITLE = 'title' +ORDER_BY_LAST_CHANGE = 'last_change' diff --git a/src/layman/common/prime_db_schema/migrate_test.py b/src/layman/common/prime_db_schema/migrate_test.py index 0d179859a..00320a630 100644 --- a/src/layman/common/prime_db_schema/migrate_test.py +++ b/src/layman/common/prime_db_schema/migrate_test.py @@ -6,7 +6,6 @@ from layman import settings, app, util, upgrade from layman.layer import LAYER_TYPE from layman.map import MAP_TYPE -from layman import upgrade from . import model, publications as pub_util, workspaces as workspaces_util from .schema_initialization import ensure_schema from .util import run_query, run_statement @@ -30,6 +29,7 @@ def save_upgrade_status(): upgrade.upgrade_v1_10.alter_schema() upgrade.upgrade_v1_10.update_style_type_in_db() upgrade.upgrade_v1_12.adjust_prime_db_schema_for_fulltext_search() + upgrade.upgrade_v1_12.adjust_prime_db_schema_for_last_change_search() upgrade.set_current_data_version(current_version) diff --git a/src/layman/common/prime_db_schema/publications.py b/src/layman/common/prime_db_schema/publications.py index 4a7749208..870bf1e14 100644 --- a/src/layman/common/prime_db_schema/publications.py +++ b/src/layman/common/prime_db_schema/publications.py @@ -50,6 +50,7 @@ def get_publication_infos(workspace_name=None, pub_type=None, style_type=None, order_by_definition = { consts.ORDER_BY_FULL_TEXT: ('ts_rank_cd(_prime_schema.my_unaccent(p.title), to_tsquery(unaccent(%s))) DESC', (ordering_full_text,)), consts.ORDER_BY_TITLE: ('unaccent(p.title) ASC', tuple()), + consts.ORDER_BY_LAST_CHANGE: ('updated_at DESC', tuple()), } assert all(ordering_item in order_by_definition.keys() for ordering_item in order_by_list) @@ -64,6 +65,7 @@ def get_publication_infos(workspace_name=None, pub_type=None, style_type=None, p.title, p.uuid::text, p.style_type, + p.updated_at, (select rtrim(concat(case when u.id is not null then w.name || ',' end, string_agg(w2.name, ',') || ',', case when p.everyone_can_read then %s || ',' end @@ -129,10 +131,11 @@ def get_publication_infos(workspace_name=None, pub_type=None, style_type=None, 'uuid': uuid, 'type': type, 'style_type': style_type, + 'updated_at': updated_at, 'access_rights': {'read': [x for x in can_read_users.split(',')], 'write': [x for x in can_write_users.split(',')]} } - for id_publication, workspace_name, type, publication_name, title, uuid, style_type, can_read_users, can_write_users + for id_publication, workspace_name, type, publication_name, title, uuid, style_type, updated_at, can_read_users, can_write_users in values} return infos @@ -224,8 +227,8 @@ def insert_publication(workspace_name, info): check_publication_info(workspace_name, info) insert_publications_sql = f'''insert into {DB_SCHEMA}.publications as p - (id_workspace, name, title, type, uuid, style_type, everyone_can_read, everyone_can_write) values - (%s, %s, %s, %s, %s, %s, %s, %s) + (id_workspace, name, title, type, uuid, style_type, everyone_can_read, everyone_can_write, updated_at) values + (%s, %s, %s, %s, %s, %s, %s, %s, current_timestamp) returning id ;''' @@ -286,7 +289,8 @@ def update_publication(workspace_name, info): title = coalesce(%s, title), style_type = coalesce(%s, style_type), everyone_can_read = coalesce(%s, everyone_can_read), - everyone_can_write = coalesce(%s, everyone_can_write) + everyone_can_write = coalesce(%s, everyone_can_write), + updated_at = current_timestamp where id_workspace = %s and name = %s and type = %s diff --git a/src/layman/common/prime_db_schema/publications_test.py b/src/layman/common/prime_db_schema/publications_test.py index 7c6314cfd..96ec6e856 100644 --- a/src/layman/common/prime_db_schema/publications_test.py +++ b/src/layman/common/prime_db_schema/publications_test.py @@ -245,6 +245,13 @@ def provide_data(self): (workspace1, MAP_TYPE, 'test_select_publications_publication1e'), (workspace2, MAP_TYPE, 'test_select_publications_publication2e'), ]), + ({'order_by_list': ['last_change'], }, [ + (workspace2, MAP_TYPE, 'test_select_publications_publication2o'), + (workspace2, MAP_TYPE, 'test_select_publications_publication2e'), + (workspace1, MAP_TYPE, 'test_select_publications_publication1oe'), + (workspace1, MAP_TYPE, 'test_select_publications_publication1o'), + (workspace1, MAP_TYPE, 'test_select_publications_publication1e'), + ]), ]) @pytest.mark.usefixtures('liferay_mock', 'ensure_layman', 'provide_data') def test_get_publications(self, query_params, expected_publications): diff --git a/src/layman/common/rest.py b/src/layman/common/rest.py index 58f9846d6..8681c2419 100644 --- a/src/layman/common/rest.py +++ b/src/layman/common/rest.py @@ -102,8 +102,9 @@ def setup_post_access_rights(request_form, kwargs, actor_name): kwargs['access_rights'][type] = access_rights -def get_publications(publication_type, user, request_args): - known_order_by_values = [consts.ORDER_BY_TITLE, consts.ORDER_BY_FULL_TEXT, ] +def get_publications(publication_type, user, request_args=None, workspace=None): + request_args = request_args or {} + known_order_by_values = [consts.ORDER_BY_TITLE, consts.ORDER_BY_FULL_TEXT, consts.ORDER_BY_LAST_CHANGE, ] full_text_filter = None if consts.FILTER_FULL_TEXT in request_args: @@ -128,6 +129,7 @@ def get_publications(publication_type, user, request_args): order_by_list = [consts.ORDER_BY_FULL_TEXT] publication_infos_whole = layman_util.get_publication_infos(publ_type=publication_type, + workspace=workspace, context={'actor_name': user, 'access_type': 'read' }, @@ -144,6 +146,7 @@ def get_publications(publication_type, user, request_args): 'url': layman_util.get_workspace_publication_url(publication_type, workspace, name), 'uuid': info["uuid"], 'access_rights': info['access_rights'], + 'updated_at': info['updated_at'].isoformat(), } for (workspace, _, name), info in publication_infos_whole.items() ] diff --git a/src/layman/common/util.py b/src/layman/common/util.py index 20c197cd7..47fefa0c5 100644 --- a/src/layman/common/util.py +++ b/src/layman/common/util.py @@ -20,4 +20,5 @@ def clear_publication_info(info): del info[key] except KeyError: pass + info['updated_at'] = info['updated_at'].isoformat() return info diff --git a/src/layman/layer/rest_layers.py b/src/layman/layer/rest_layers.py index a7cea8bae..5e74bcb14 100644 --- a/src/layman/layer/rest_layers.py +++ b/src/layman/layer/rest_layers.py @@ -22,4 +22,4 @@ def get(): app.logger.info(f"GET Layers, user={g.user}") user = get_authn_username() or settings.ANONYM_USER - return rest_common.get_publications(LAYER_TYPE, user, request.args) + return rest_common.get_publications(LAYER_TYPE, user, request_args=request.args) diff --git a/src/layman/layer/rest_workspace_layers.py b/src/layman/layer/rest_workspace_layers.py index 93435afea..fee7cfef3 100644 --- a/src/layman/layer/rest_workspace_layers.py +++ b/src/layman/layer/rest_workspace_layers.py @@ -7,9 +7,9 @@ from layman import settings, authn, util as layman_util from . import util, LAYER_TYPE, LAYER_REST_PATH_NAME from .filesystem import input_file, input_style, input_chunk, uuid -from layman.authn import authenticate +from layman.authn import authenticate, get_authn_username from layman.authz import authorize_workspace_publications_decorator -from layman.common import redis as redis_util +from layman.common import redis as redis_util, rest as rest_common bp = Blueprint('rest_workspace_layers', __name__) @@ -32,21 +32,8 @@ def after_request(response): def get(username): app.logger.info(f"GET Layers, user={g.user}") - layer_infos_whole = layman_util.get_publication_infos(username, LAYER_TYPE) - - infos = [ - { - 'name': info["name"], - 'workspace': workspace, - 'title': info.get("title", None), - 'url': url_for('rest_workspace_layer.get', layername=name, username=username), - 'uuid': info["uuid"], - 'access_rights': info['access_rights'], - } - for (workspace, publication_type, name), info in layer_infos_whole.items() - ] - sorted_infos = sorted(infos, key=lambda x: x['name']) - return jsonify(sorted_infos), 200 + user = get_authn_username() or settings.ANONYM_USER + return rest_common.get_publications(LAYER_TYPE, user, workspace=username) @bp.route(f"/{LAYER_REST_PATH_NAME}", methods=['POST']) diff --git a/src/layman/map/rest_maps.py b/src/layman/map/rest_maps.py index c58a2f6fc..20defc836 100644 --- a/src/layman/map/rest_maps.py +++ b/src/layman/map/rest_maps.py @@ -22,4 +22,4 @@ def get(): app.logger.info(f"GET Maps, user={g.user}") user = get_authn_username() or settings.ANONYM_USER - return rest_common.get_publications(MAP_TYPE, user, request.args) + return rest_common.get_publications(MAP_TYPE, user, request_args=request.args) diff --git a/src/layman/map/rest_workspace_maps.py b/src/layman/map/rest_workspace_maps.py index 8e0ea6759..72a5f1ca7 100644 --- a/src/layman/map/rest_workspace_maps.py +++ b/src/layman/map/rest_workspace_maps.py @@ -9,10 +9,10 @@ from layman.util import check_username_decorator, url_for from . import util, MAP_TYPE, MAP_REST_PATH_NAME from .filesystem import input_file, uuid -from layman import authn, util as layman_util -from layman.authn import authenticate +from layman import authn, util as layman_util, settings +from layman.authn import authenticate, get_authn_username from layman.authz import authorize_workspace_publications_decorator -from layman.common import redis as redis_util +from layman.common import redis as redis_util, rest as rest_common bp = Blueprint('rest_workspace_maps', __name__) @@ -35,21 +35,8 @@ def after_request(response): def get(username): app.logger.info(f"GET Maps, user={g.user}") - mapinfos_whole = layman_util.get_publication_infos(username, MAP_TYPE) - - infos = [ - { - 'name': info["name"], - 'workspace': workspace, - 'title': info.get("title", None), - 'url': url_for('rest_workspace_map.get', mapname=name, username=username), - 'uuid': info['uuid'], - 'access_rights': info['access_rights'], - } - for (workspace, publication_type, name), info in mapinfos_whole.items() - ] - sorted_infos = sorted(infos, key=lambda x: x['name']) - return jsonify(sorted_infos), 200 + user = get_authn_username() or settings.ANONYM_USER + return rest_common.get_publications(MAP_TYPE, user, workspace=username) @bp.route(f"/{MAP_REST_PATH_NAME}", methods=['POST']) diff --git a/src/layman/rest_multipublication_test.py b/src/layman/rest_multipublication_test.py index 9b232cf61..e140f6e79 100644 --- a/src/layman/rest_multipublication_test.py +++ b/src/layman/rest_multipublication_test.py @@ -122,6 +122,10 @@ def provide_data(self): (workspace1, 'test_get_publications_publication1e'), (workspace2, 'test_get_publications_publication2e'), ],), + (authn_headers_user2, {'order_by': 'last_change'}, [(workspace2, 'test_get_publications_publication2o'), + (workspace2, 'test_get_publications_publication2e'), + (workspace1, 'test_get_publications_publication1e'), + ],), ]) @pytest.mark.parametrize('publication_type', process_client.PUBLICATION_TYPES) @pytest.mark.usefixtures('liferay_mock', 'ensure_layman', 'provide_data') diff --git a/src/layman/rest_publication_test.py b/src/layman/rest_publication_test.py index 7db6ba7de..03b2de021 100644 --- a/src/layman/rest_publication_test.py +++ b/src/layman/rest_publication_test.py @@ -1,9 +1,14 @@ import pytest +import datetime +from dateutil.parser import parse from test import process_client -from layman.http import LaymanError +from layman import LaymanError, settings, app +from layman.common.prime_db_schema import util as db_util from layman.common.micka import util as micka_util +db_schema = settings.LAYMAN_PRIME_SCHEMA + @pytest.mark.parametrize('publ_type', process_client.PUBLICATION_TYPES) @pytest.mark.usefixtures('ensure_layman') @@ -105,3 +110,51 @@ def test_soap_authz(self, publ_type, params_and_expected_list): anon_number_of_records = micka_util.get_number_of_records(publ_muuid, use_authn=False) assert bool(anon_number_of_records) == anonymous_visibility, \ f"muuid={publ_muuid}, access_rights={access_rights}, number_of_records={anon_number_of_records}" + + +@pytest.mark.parametrize('publication_type', process_client.PUBLICATION_TYPES) +@pytest.mark.usefixtures('ensure_layman') +def test_updated_at(publication_type): + workspace = 'test_update_at_workspace' + publication = 'test_update_at_publication' + + query = f''' + select p.updated_at + from {db_schema}.publications p inner join + {db_schema}.workspaces w on p.id_workspace = w.id + where w.name = %s + and p.type = %s + and p.name = %s + ;''' + + timestamp1 = datetime.datetime.now(datetime.timezone.utc) + process_client.publish_workspace_publication(publication_type, workspace, publication) + timestamp2 = datetime.datetime.now(datetime.timezone.utc) + + with app.app_context(): + results = db_util.run_query(query, (workspace, publication_type, publication)) + assert len(results) == 1 and len(results[0]) == 1, results + updated_at_db = results[0][0] + assert timestamp1 < updated_at_db < timestamp2 + + info = process_client.get_workspace_publication(publication_type, workspace, publication) + updated_at_rest_str = info['updated_at'] + updated_at_rest = parse(updated_at_rest_str) + assert timestamp1 < updated_at_rest < timestamp2 + + timestamp3 = datetime.datetime.now(datetime.timezone.utc) + process_client.patch_workspace_publication(publication_type, workspace, publication, title='Title') + timestamp4 = datetime.datetime.now(datetime.timezone.utc) + + with app.app_context(): + results = db_util.run_query(query, (workspace, publication_type, publication)) + assert len(results) == 1 and len(results[0]) == 1, results + updated_at_db = results[0][0] + assert timestamp3 < updated_at_db < timestamp4 + + info = process_client.get_workspace_publication(publication_type, workspace, publication) + updated_at_rest_str = info['updated_at'] + updated_at_rest = parse(updated_at_rest_str) + assert timestamp3 < updated_at_rest < timestamp4 + + process_client.delete_workspace_publication(publication_type, workspace, publication) diff --git a/src/layman/upgrade/__init__.py b/src/layman/upgrade/__init__.py index 6c3de39e0..445f82d47 100644 --- a/src/layman/upgrade/__init__.py +++ b/src/layman/upgrade/__init__.py @@ -21,6 +21,7 @@ upgrade_v1_10.update_style_type_in_db, ]), ((1, 12, 0), [upgrade_v1_12.adjust_prime_db_schema_for_fulltext_search, + upgrade_v1_12.adjust_prime_db_schema_for_last_change_search, ]) ] diff --git a/src/layman/upgrade/upgrade_v1_10.py b/src/layman/upgrade/upgrade_v1_10.py index 2270d59bb..f83231e59 100644 --- a/src/layman/upgrade/upgrade_v1_10.py +++ b/src/layman/upgrade/upgrade_v1_10.py @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) +db_schema = settings.LAYMAN_PRIME_SCHEMA + def alter_schema(): logger.info(f' Starting - alter DB prime schema') @@ -66,8 +68,20 @@ def check_workspace_names(): def migrate_layers_to_wms_workspace(workspace=None): logger.info(f' Starting - migrate layers to WMS workspace') - infos = util.get_publication_infos(publ_type=LAYER_TYPE, workspace=workspace) - for (workspace, publication_type, layer) in infos.keys(): + query = f''' +select w.name, + p.type, + p.name +from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace +where p.type = %s +''' + params = (LAYER_TYPE, ) + if workspace: + query = query + ' AND w.name = %s' + params = params + (workspace, ) + publications = db_util.run_query(query, params) + for (workspace, publication_type, layer) in publications: logger.info(f' Migrate layer {workspace}.{layer}') info = util.get_publication_info(workspace, publication_type, layer) geoserver_workspace = wms.get_geoserver_workspace(workspace) @@ -113,12 +127,20 @@ def migrate_layers_to_wms_workspace(workspace=None): def migrate_maps_on_wms_workspace(): logger.info(f' Starting - migrate maps json urls') - infos = util.get_publication_infos(publ_type=MAP_TYPE) + query = f''' + select w.name, + p.name + from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace + where p.type = %s + ''' + params = (MAP_TYPE,) + publications = db_util.run_query(query, params) gs_url = gs_util.get_gs_proxy_base_url() gs_url = gs_url if gs_url.endswith('/') else f"{gs_url}/" gs_wms_url_pattern = r'^' + re.escape(gs_url) + r'(' + util.USERNAME_ONLY_PATTERN + r')' + r'(/(?:ows|wms|wfs).*)$' all_workspaces = workspaces.get_workspace_names() - for (workspace, _, map) in infos.keys(): + for (workspace, map) in publications: file_path = input_file.get_map_file(workspace, map) is_changed = False with open(file_path, 'r') as map_file: @@ -150,8 +172,19 @@ def migrate_maps_on_wms_workspace(): def migrate_metadata_records(workspace=None): logger.info(f' Starting - migrate publication metadata records') - infos = util.get_publication_infos(publ_type=LAYER_TYPE, workspace=workspace) - for (workspace, _, layer) in infos.keys(): + query = f''' + select w.name, + p.name + from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace + where p.type = %s + ''' + params = (LAYER_TYPE,) + if workspace: + query = query + ' AND w.name = %s' + params = params + (workspace,) + publications = db_util.run_query(query, params) + for (workspace, layer) in publications: wms.clear_cache(workspace) logger.info(f' Migrate layer {workspace}.{layer}') try: @@ -168,8 +201,19 @@ def migrate_metadata_records(workspace=None): f' WMS URL was not migrated (should be {exp_wms_url}, but is {md_wms_url})!') time.sleep(0.5) - infos = util.get_publication_infos(publ_type=MAP_TYPE, workspace=workspace) - for (workspace, _, map) in infos.keys(): + query = f''' + select w.name, + p.name + from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace + where p.type = %s + ''' + params = (MAP_TYPE,) + if workspace: + query = query + ' AND w.name = %s' + params = params + (workspace,) + publications = db_util.run_query(query, params) + for (workspace, map) in publications: logger.info(f' Migrate map {workspace}.{map}') try: muuid = map_csw.patch_map(workspace, map, ['graphic_url', 'identifier', 'map_endpoint', 'map_file_endpoint', ], @@ -189,8 +233,16 @@ def migrate_metadata_records(workspace=None): def migrate_input_sld_directory_to_input_style(): logger.info(f' Starting - migrate input_sld directories to input_style') - infos = util.get_publication_infos(publ_type=LAYER_TYPE) - for (workspace, _, layer) in infos.keys(): + query = f''' + select w.name, + p.name + from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace + where p.type = %s + ''' + params = (LAYER_TYPE,) + publications = db_util.run_query(query, params) + for (workspace, layer) in publications: sld_path = os.path.join(layer_fs_util.get_layer_dir(workspace, layer), 'input_sld') if os.path.exists(sld_path): diff --git a/src/layman/upgrade/upgrade_v1_12.py b/src/layman/upgrade/upgrade_v1_12.py index ad99b8312..1c944da61 100644 --- a/src/layman/upgrade/upgrade_v1_12.py +++ b/src/layman/upgrade/upgrade_v1_12.py @@ -1,12 +1,54 @@ +import os +import datetime + +from layman import settings from layman.common.prime_db_schema import util as db_util +db_schema = settings.LAYMAN_PRIME_SCHEMA + def adjust_prime_db_schema_for_fulltext_search(): - statement = '''CREATE EXTENSION IF NOT EXISTS unaccent; -drop index if exists _prime_schema.title_tsv_idx; -drop function if exists _prime_schema.my_unaccent; + statement = f'''CREATE EXTENSION IF NOT EXISTS unaccent; + drop index if exists {db_schema}.title_tsv_idx; + drop function if exists {db_schema}.my_unaccent; + + CREATE FUNCTION {db_schema}.my_unaccent(text) RETURNS tsvector LANGUAGE SQL IMMUTABLE AS 'SELECT to_tsvector(unaccent($1))'; + CREATE INDEX title_tsv_idx ON {db_schema}.publications USING GIST ({db_schema}.my_unaccent(title)); + ''' + + db_util.run_statement(statement) + + +def adjust_prime_db_schema_for_last_change_search(): + statement = f'ALTER TABLE {db_schema}.publications ADD COLUMN IF NOT EXISTS updated_at timestamp with time zone;' + db_util.run_statement(statement) + + query = f'''select p.id, + w.name, + p.type, + p.name +from {db_schema}.publications p inner join + {db_schema}.workspaces w on w.id = p.id_workspace +;''' + publications = db_util.run_query(query) + for (id, workspace, type, name, ) in publications: + publ_dir = os.path.join( + settings.LAYMAN_DATA_DIR, + 'users', + workspace, + type.split('.')[1] + 's', + name, + ) + updated_at = None + for root, _, files in os.walk(publ_dir): + for file in files: + file_updated_at = os.stat(os.path.join(root, file)).st_mtime + updated_at = max(updated_at, file_updated_at) if updated_at else file_updated_at + updated_at = datetime.datetime.fromtimestamp(updated_at, datetime.timezone.utc)\ + if updated_at else datetime.datetime.now(datetime.timezone.utc) + + update = f'update {db_schema}.publications set updated_at = %s where id = %s;' + db_util.run_statement(update, (updated_at, id, )) -CREATE FUNCTION _prime_schema.my_unaccent(text) RETURNS tsvector LANGUAGE SQL IMMUTABLE AS 'SELECT to_tsvector(unaccent($1))'; -CREATE INDEX title_tsv_idx ON _prime_schema.publications USING GIST (_prime_schema.my_unaccent(title)); -''' + statement = f'ALTER TABLE {db_schema}.publications ALTER COLUMN updated_at SET NOT NULL;' db_util.run_statement(statement) diff --git a/src/layman/upgrade/upgrade_v1_12_test.py b/src/layman/upgrade/upgrade_v1_12_test.py new file mode 100644 index 000000000..d7df9c374 --- /dev/null +++ b/src/layman/upgrade/upgrade_v1_12_test.py @@ -0,0 +1,56 @@ +import pytest +import datetime + + +from layman import app, settings +from . import upgrade_v1_12 +from test import process_client +from layman.common.prime_db_schema import util as db_util + +db_schema = settings.LAYMAN_PRIME_SCHEMA + + +@pytest.mark.usefixtures('ensure_layman') +def test_adjust_prime_db_schema_for_last_change_search(): + workspace = 'test_adjust_prime_db_schema_for_last_change_search_workspace' + layer = 'test_adjust_prime_db_schema_for_last_change_search_layer' + map = 'test_adjust_prime_db_schema_for_last_change_search_map' + + timestamp1 = datetime.datetime.now(datetime.timezone.utc) + process_client.publish_workspace_layer(workspace, layer) + process_client.publish_workspace_map(workspace, map) + timestamp2 = datetime.datetime.now(datetime.timezone.utc) + with app.app_context(): + statement = f'ALTER TABLE {db_schema}.publications ALTER COLUMN updated_at DROP NOT NULL;' + db_util.run_statement(statement) + statement = f'update {db_schema}.publications set updated_at = null;' + db_util.run_statement(statement) + + query = f'select p.id from {db_schema}.publications p where p.updated_at is not null;' + results = db_util.run_query(query) + assert not results, results + + upgrade_v1_12.adjust_prime_db_schema_for_last_change_search() + + query = f''' +select p.updated_at +from {db_schema}.publications p inner join + {db_schema}.workspaces w on p.id_workspace = w.id +where w.name = %s + and p.type = %s + and p.name = %s +;''' + results = db_util.run_query(query, (workspace, 'layman.layer', layer)) + assert len(results) == 1 and len(results[0]) == 1, results + layer_updated_at = results[0][0] + assert timestamp1 < layer_updated_at < timestamp2 + + results = db_util.run_query(query, (workspace, 'layman.map', map)) + assert len(results) == 1 and len(results[0]) == 1, results + map_updated_at = results[0][0] + assert timestamp1 < map_updated_at < timestamp2 + + assert layer_updated_at < map_updated_at + + process_client.delete_workspace_layer(workspace, layer) + process_client.delete_workspace_map(workspace, map) diff --git a/src/layman/util_test.py b/src/layman/util_test.py index c0644a97a..b3682f256 100644 --- a/src/layman/util_test.py +++ b/src/layman/util_test.py @@ -233,6 +233,8 @@ def test_get_publication_infos(publication_type): for publication_name in publication_infos: if publication_infos[publication_name].get('id'): del publication_infos[publication_name]['id'] + if publication_infos[publication_name].get('updated_at'): + del publication_infos[publication_name]['updated_at'] assert publication_infos == expected_result, (publication_infos, expected_result) process_client.delete_workspace_publication(publication_type, workspace, publication)