Filtering by full text search

.env.demo

@@ -60,7 +60,7 @@ LAYMAN_CLIENT_URL=http://layman_client:3000/client/
 
 # client
 LAYMAN_CLIENT_PUBLIC_URL=http://localhost/client/
-LAYMAN_CLIENT_VERSION=v1.6.1
+LAYMAN_CLIENT_VERSION=89ae01037056b9043a23f271a132eb19b103680b
 
 # extra hosts to be added to /etc/hosts
 EXTRA_HOST1=1.2.3.4:1.2.3.4

.env.dev

@@ -62,7 +62,7 @@ LAYMAN_CLIENT_URL=http://layman_client:3000/client/
 
 # client
 LAYMAN_CLIENT_PUBLIC_URL=http://localhost:3000/client/
-LAYMAN_CLIENT_VERSION=v1.6.1
+LAYMAN_CLIENT_VERSION=89ae01037056b9043a23f271a132eb19b103680b
 
 
 ##############################################################################

.env.test

@@ -62,7 +62,7 @@ LAYMAN_CLIENT_URL=http://layman_client_test:3000/client/
 
 # client
 LAYMAN_CLIENT_PUBLIC_URL=http://layman_test_run_1:8000/client/
-LAYMAN_CLIENT_VERSION=v1.6.1
+LAYMAN_CLIENT_VERSION=89ae01037056b9043a23f271a132eb19b103680b
 
 
 ##############################################################################

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## v1.12.0
+ {release-date}
+### Upgrade requirements
+
+### Migrations and checks
+
+### Changes
+- [#273](https://github.com/jirik/layman/issues/273) Endpoints [GET Layers](doc/rest.md#get-layers) and [GET Maps](doc/rest.md#get-maps) can filter and reorder results according to new query parameters. 
+
 ## v1.11.0
  2021-03-16
 ### Upgrade requirements

doc/rest.md

@@ -34,7 +34,9 @@
 Get list of published layers.
 
 #### Request
-No action parameters.
+Query parameters:
+- *full_text_filter*: String. List of words separated by space. Only layers with at least one of them in title will be return. Search is case-insensitive, unaccent and did lemmatization for English. By default, layers are ordered by search rank in response if this filter is used.
+
 #### Response
 Content-Type: `application/json`
 
@@ -390,7 +392,9 @@ JSON object with one attribute:
 Get list of published maps (map compositions).
 
 #### Request
-No action parameters.
+Query parameters:
+- *full_text_filter*: String. List of words separated by space. Only maps with at least one of them in title will be return. Search is case-insensitive, unaccent and did lemmatization for English. By default, maps are ordered by search rank in response if this filter is used.
+
 #### Response
 Content-Type: `application/json`
 

src/layman/common/prime_db_schema/migrate_test.py

@@ -28,6 +28,9 @@ def save_upgrade_status():
     with app.app_context():
         upgrade.upgrade_v1_9.initialize_data_versioning()
         upgrade.upgrade_v1_10.alter_schema()
+        upgrade.upgrade_v1_10.update_style_type_in_db()
+        upgrade.upgrade_v1_12.install_unaccent_to_db()
+
         upgrade.set_current_data_version(current_version)
 
 

src/layman/common/prime_db_schema/publications.py

@@ -11,8 +11,50 @@
 psycopg2.extras.register_uuid()
 
 
-def get_publication_infos(workspace_name=None, pub_type=None, style_type=None, reader=None, writer=None):
-    sql_basic = f"""
+def get_publication_infos(workspace_name=None, pub_type=None, style_type=None,
+                          reader=None, writer=None,
+                          full_text_filter=None,
+                          order_by_list=None,
+                          ordering_full_text=None,
+                          ):
+    order_by_list = order_by_list or []
+
+    where_params_def = [
+        (workspace_name, 'w.name = %s', (workspace_name,)),
+        (pub_type, 'p.type = %s', (pub_type,)),
+        (style_type, 'p.style_type::text = %s', (style_type,)),
+        (reader == settings.ANONYM_USER, 'p.everyone_can_read = TRUE', tuple()),
+        (reader and reader != settings.ANONYM_USER, f"""(p.everyone_can_read = TRUE
+                        or (u.id is not null and w.name = %s)
+                        or EXISTS(select 1
+                                  from {DB_SCHEMA}.rights r inner join
+                                       {DB_SCHEMA}.users u2 on r.id_user = u2.id inner join
+                                       {DB_SCHEMA}.workspaces w2 on w2.id = u2.id_workspace
+                                  where r.id_publication = p.id
+                                    and r.type = 'read'
+                                    and w2.name = %s))""", (reader, reader,)),
+        (writer == settings.ANONYM_USER, 'p.everyone_can_write = TRUE', tuple()),
+        (writer and writer != settings.ANONYM_USER, f"""(p.everyone_can_write = TRUE
+                        or (u.id is not null and w.name = %s)
+                        or EXISTS(select 1
+                                  from {DB_SCHEMA}.rights r inner join
+                                       {DB_SCHEMA}.users u2 on r.id_user = u2.id inner join
+                                       {DB_SCHEMA}.workspaces w2 on w2.id = u2.id_workspace
+                                  where r.id_publication = p.id
+                                    and r.type = 'write'
+                                    and w2.name = %s))""", (writer, writer,)),
+        (full_text_filter, '_prime_schema.my_unaccent(p.title) @@ to_tsquery(unaccent(%s))', (full_text_filter,)),
+    ]
+
+    order_by_definition = {
+        'full_text': ('ts_rank_cd(_prime_schema.my_unaccent(p.title), to_tsquery(unaccent(%s))) DESC', (ordering_full_text,)),
+    }
+
+    assert all(ordering_item in order_by_definition.keys() for ordering_item in order_by_list)
+
+    #########################################################
+    # SELECT clause
+    select_clause = f"""
 select p.id as id_publication,
        w.name as workspace_name,
        p.type,
@@ -42,44 +84,40 @@ def get_publication_infos(workspace_name=None, pub_type=None, style_type=None, r
      {DB_SCHEMA}.publications p on p.id_workspace = w.id left join
      {DB_SCHEMA}.users u on u.id_workspace = w.id
 """
-    query_params = (ROLE_EVERYONE, ROLE_EVERYONE, )
-    where_parts = []
-    where_params_def = [(workspace_name, 'w.name = %s', (workspace_name, )),
-                        (pub_type, 'p.type = %s', (pub_type, )),
-                        (style_type, 'p.style_type::text = %s', (style_type, )),
-                        (reader == settings.ANONYM_USER, 'p.everyone_can_read = TRUE', tuple()),
-                        (reader and reader != settings.ANONYM_USER, f"""(p.everyone_can_read = TRUE
-                        or (u.id is not null and w.name = %s)
-                        or EXISTS(select 1
-                                  from {DB_SCHEMA}.rights r inner join
-                                       {DB_SCHEMA}.users u2 on r.id_user = u2.id inner join
-                                       {DB_SCHEMA}.workspaces w2 on w2.id = u2.id_workspace
-                                  where r.id_publication = p.id
-                                    and r.type = 'read'
-                                    and w2.name = %s))""", (reader, reader,)),
-                        (writer == settings.ANONYM_USER, 'p.everyone_can_write = TRUE', tuple()),
-                        (writer and writer != settings.ANONYM_USER, f"""(p.everyone_can_write = TRUE
-                        or (u.id is not null and w.name = %s)
-                        or EXISTS(select 1
-                                  from {DB_SCHEMA}.rights r inner join
-                                       {DB_SCHEMA}.users u2 on r.id_user = u2.id inner join
-                                       {DB_SCHEMA}.workspaces w2 on w2.id = u2.id_workspace
-                                  where r.id_publication = p.id
-                                    and r.type = 'write'
-                                    and w2.name = %s))""", (writer, writer,)),
-                        ]
+    select_params = (ROLE_EVERYONE, ROLE_EVERYONE, )
+
+    #########################################################
+    # WHERE clause
+    where_params = tuple()
+    where_parts = list()
     for (value, where_part, params, ) in where_params_def:
         if value:
             where_parts.append(where_part)
-            query_params = query_params + params
-
+            where_params = where_params + params
     where_clause = ''
     if where_parts:
         where_clause = 'WHERE ' + '\n  AND '.join(where_parts) + '\n'
 
-    order_by_clause = 'ORDER BY w.name ASC, p.name ASC\n'
-    select = sql_basic + where_clause + order_by_clause
-    values = util.run_query(select, query_params)
+    #########################################################
+    # ORDER BY clause
+    order_by_params = tuple()
+    order_by_parts = list()
+    for order_by_part in order_by_list:
+        order_by_parts.append(order_by_definition[order_by_part][0])
+        order_by_params = order_by_params + order_by_definition[order_by_part][1]
+
+    order_by_parts.append('w.name ASC')
+    order_by_parts.append('p.name ASC')
+    order_by_clause = 'ORDER BY ' + ', '.join(order_by_parts)
+
+    #########################################################
+    # Put it together
+    sql_params = select_params + where_params + order_by_params
+    select = select_clause + where_clause + order_by_clause
+    values = util.run_query(select, sql_params)
+
+    # print(f'get_publication_infos:\n\n order_by_clause={order_by_clause},\n where_clause={where_clause},\n sql_params={sql_params},'
+    #       f'\n order_by_list={order_by_list},\n full_text_ordering={full_text_ordering}')
 
     infos = {(workspace_name,
               type,

src/layman/common/prime_db_schema/publications_test.py

@@ -151,22 +151,27 @@ class TestSelectPublicationsComplex:
     publications = [
         (workspace1, MAP_TYPE, 'test_select_publications_publication1e',
          {'headers': authn_headers_user1,
+          'title': 'Příliš žluťoučký Kůň úpěl ďábelské ódy',
           'access_rights': {'read': settings.RIGHTS_EVERYONE_ROLE,
                             'write': settings.RIGHTS_EVERYONE_ROLE}, }),
         (workspace1, MAP_TYPE, 'test_select_publications_publication1o',
          {'headers': authn_headers_user1,
+          'title': 'kun Karel',
           'access_rights': {'read': workspace1,
                             'write': workspace1}, }),
         (workspace1, MAP_TYPE, 'test_select_publications_publication1oe',
          {'headers': authn_headers_user1,
+          'title': 'jedna dva tři čtyři',
           'access_rights': {'read': settings.RIGHTS_EVERYONE_ROLE,
                             'write': workspace1}, }),
         (workspace2, MAP_TYPE, 'test_select_publications_publication2e',
          {'headers': authn_headers_user2,
+          'title': 'Svíčky is the best game',
           'access_rights': {'read': settings.RIGHTS_EVERYONE_ROLE,
                             'write': settings.RIGHTS_EVERYONE_ROLE}, }),
         (workspace2, MAP_TYPE, 'test_select_publications_publication2o',
          {'headers': authn_headers_user2,
+          'title': 'druhá mapa JeDnA óda',
           'access_rights': {'read': workspace2,
                             'write': workspace2}, }),
     ]
@@ -204,12 +209,42 @@ def provide_data(self):
                                   (workspace2, MAP_TYPE, 'test_select_publications_publication2e'),
                                   (workspace2, MAP_TYPE, 'test_select_publications_publication2o'),
                                   ]),
+        ({'full_text_filter': 'dva'}, [(workspace1, MAP_TYPE, 'test_select_publications_publication1oe'),
+                                       ]),
+        ({'full_text_filter': 'games'}, [(workspace2, MAP_TYPE, 'test_select_publications_publication2e'),
+                                         ]),
+        ({'full_text_filter': 'kun'}, [(workspace1, MAP_TYPE, 'test_select_publications_publication1e'),
+                                       (workspace1, MAP_TYPE, 'test_select_publications_publication1o'),
+                                       ]),
+        ({'full_text_filter': 'jedna'}, [(workspace1, MAP_TYPE, 'test_select_publications_publication1oe'),
+                                         (workspace2, MAP_TYPE, 'test_select_publications_publication2o'),
+                                         ]),
+        ({'full_text_filter': 'upet'}, []),
+        ({'full_text_filter': 'dva | kun'}, [(workspace1, MAP_TYPE, 'test_select_publications_publication1e'),
+                                             (workspace1, MAP_TYPE, 'test_select_publications_publication1o'),
+                                             (workspace1, MAP_TYPE, 'test_select_publications_publication1oe'),
+                                             ]),
+        ({'full_text_filter': 'kun & ody'}, [(workspace1, MAP_TYPE, 'test_select_publications_publication1e'),
+                                             ]),
+        ({'order_by_list': ['full_text'], 'ordering_full_text': 'jedna'}, [
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1oe'),
+            (workspace2, MAP_TYPE, 'test_select_publications_publication2o'),
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1e'),
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1o'),
+            (workspace2, MAP_TYPE, 'test_select_publications_publication2e'),
+        ]),
+        ({'full_text_filter': 'dva | kun', 'order_by_list': ['full_text'], 'ordering_full_text': 'karel | kun'}, [
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1o'),
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1e'),
+            (workspace1, MAP_TYPE, 'test_select_publications_publication1oe'),
+        ]),
     ])
     @pytest.mark.usefixtures('liferay_mock', 'ensure_layman', 'provide_data')
     def test_get_publications(self, query_params, expected_publications):
         with app.app_context():
             infos = publications.get_publication_infos(**query_params)
         info_publications = list(infos.keys())
+        assert set(expected_publications) == set(info_publications)
         assert expected_publications == info_publications
 
 

src/layman/common/prime_db_schema/util.py

@@ -1,5 +1,6 @@
 from flask import g, current_app as app
 import psycopg2
+import re
 
 from layman import settings
 from layman.http import LaymanError
@@ -59,3 +60,9 @@ def run_statement(query, data=None, conn_cur=None, encapsulate_exception=True):
         else:
             raise exc
     return rows
+
+
+def to_tsquery_string(value):
+    value = re.sub(r'[\W_]+', ' ', value, flags=re.UNICODE).strip()
+    value = value.replace(' ', ' | ')
+    return value

src/layman/common/prime_db_schema/util_test.py

@@ -0,0 +1,14 @@
+import pytest
+
+from . import util
+
+
+@pytest.mark.parametrize('input_string, exp_result', [
+    ('Příliš žluťoučký kůň úpěl ďábelské ódy', 'Příliš | žluťoučký | kůň | úpěl | ďábelské | ódy'),
+    (' #@ Příliš žluťoučký kůň úpěl ďábelské ódy  \n', 'Příliš | žluťoučký | kůň | úpěl | ďábelské | ódy'),
+    ('Pří_liš', 'Pří | liš'),
+    ('\'Too yellow horse\' means "Příliš žluťoučký kůň".', 'Too | yellow | horse | means | Příliš | žluťoučký | kůň'),
+    ('\tThe Fačřš_tÚŮTŤsa   "  a34432[;] ;.\\Ra\'\'ts  ', 'The | Fačřš | tÚŮTŤsa | a34432 | Ra | ts'),
+])
+def test_to_tsquery_string(input_string, exp_result):
+    assert util.to_tsquery_string(input_string) == exp_result

src/layman/common/rest.py

@@ -1,18 +1,19 @@
+from flask import jsonify
 import re
 
-from layman import settings
+from layman import settings, util as layman_util
+from layman.common.prime_db_schema import util as prime_db_schema_util
 from .util import PUBLICATION_NAME_ONLY_PATTERN
-from layman.util import USERNAME_ONLY_PATTERN, get_publication_types
 
 
 def _get_pub_type_pattern():
-    publ_type_names = [publ_type['rest_path_name'] for publ_type in get_publication_types().values()]
+    publ_type_names = [publ_type['rest_path_name'] for publ_type in layman_util.get_publication_types().values()]
     publ_type_pattern = r"(?P<publication_type>" + "|".join(publ_type_names) + r")"
     return publ_type_pattern
 
 
 def _get_workspace_multi_publication_path_pattern():
-    workspace_pattern = r"(?P<workspace>" + USERNAME_ONLY_PATTERN + r")"
+    workspace_pattern = r"(?P<workspace>" + layman_util.USERNAME_ONLY_PATTERN + r")"
     return f"^/rest/({settings.REST_WORKSPACES_PREFIX}/)?" + workspace_pattern + "/" + _get_pub_type_pattern()
 
 
@@ -51,7 +52,7 @@ def get_url_name_to_publication_type():
     if _URL_NAME_TO_PUBLICATION_TYPE is None:
         _URL_NAME_TO_PUBLICATION_TYPE = {
             publ_type['rest_path_name']: publ_type
-            for publ_type in get_publication_types().values()
+            for publ_type in layman_util.get_publication_types().values()
         }
     return _URL_NAME_TO_PUBLICATION_TYPE
 
@@ -98,3 +99,38 @@ def setup_post_access_rights(request_form, kwargs, actor_name):
         else:
             access_rights = list({x.strip() for x in request_form['access_rights.' + type].split(',')})
         kwargs['access_rights'][type] = access_rights
+
+
+def get_publications(publication_type, user, request_args):
+
+    full_text_filter = None
+    if 'full_text_filter' in request_args:
+        full_text_filter = prime_db_schema_util.to_tsquery_string(request_args.get('full_text_filter')) or None
+
+    order_by_list = []
+    ordering_full_text = None
+    if full_text_filter:
+        ordering_full_text = full_text_filter
+        order_by_list = ['full_text']
+
+    publication_infos_whole = layman_util.get_publication_infos(publ_type=publication_type,
+                                                                context={'actor_name': user,
+                                                                         'access_type': 'read'
+                                                                         },
+                                                                full_text_filter=full_text_filter,
+                                                                order_by_list=order_by_list,
+                                                                ordering_full_text=ordering_full_text,
+                                                                )
+
+    infos = [
+        {
+            'name': name,
+            'workspace': workspace,
+            'title': info.get("title"),
+            'url': layman_util.get_workspace_publication_url(publication_type, workspace, name),
+            'uuid': info["uuid"],
+            'access_rights': info['access_rights'],
+        }
+        for (workspace, _, name), info in publication_infos_whole.items()
+    ]
+    return jsonify(infos), 200

src/layman/layer/__init__.py

@@ -102,3 +102,10 @@ def get_layer_sources():
                                 ),
                    ]
 NO_STYLE_DEF = STYLE_TYPES_DEF[0]
+
+
+from layman.util import url_for
+
+
+def get_workspace_publication_url(workspace, publication_name):
+    return url_for('rest_workspace_layer.get', layername=publication_name, username=workspace)