Feature/ogc 508 replace elastic search by postgres v2

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
     - id: sass-lint
       files: '^src/.*\.scss'
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v9.9.1
+    rev: v9.10.0
     hooks:
     - id: eslint
       files: '^src/.*\.jsx?$'

src/onegov/agency/views/search.py

@@ -1,9 +1,8 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
-from onegov.org.views.search import search as search_view
-
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search as search_view, search_postgres
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -23,3 +22,14 @@
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search.pt', permission=Public)
+def agency_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'AgencyRequest'
+) -> 'RenderData | Response':
+    data = search_postgres(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/directory/models/directory_entry.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import ContentMixin
 from onegov.core.orm.mixins import TimestampMixin
@@ -6,7 +8,7 @@
 from onegov.file import AssociatedFiles
 from onegov.gis import CoordinatesMixin
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, func, cast, ARRAY, String
 from sqlalchemy import ForeignKey
 from sqlalchemy import Index
 from sqlalchemy import Text
@@ -30,10 +32,10 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
     __tablename__ = 'directory_entries'
 
     es_properties = {
-        'keywords': {'type': 'keyword'},
         'title': {'type': 'localized'},
         'lead': {'type': 'localized'},
-        'directory_id': {'type': 'keyword'},
+        'keywords': {'type': 'keyword'},
+        # 'directory_id': {'type': 'keyword'},
 
         # since the searchable text might include html, we remove it
         # even if there's no html -> possibly decreasing the search
@@ -113,17 +115,26 @@ def external_link_visible(self) -> bool | None:
     def directory_name(self) -> str:
         return self.directory.name
 
-    @property
+    @hybrid_property
     def keywords(self) -> set[str]:
         return set(self._keywords.keys()) if self._keywords else set()
 
     # FIXME: asymmetric properties are not supported by mypy, switch to
     #        a custom descriptor, if desired.
-    @keywords.setter
+    @keywords.setter  # type:ignore[no-redef]
     def keywords(self, value: 'Collection[str] | None') -> None:
         self._keywords = dict.fromkeys(value, '') if value else None
 
-    @property
+    @keywords.expression  # type:ignore[no-redef]
+    def keywords(cls):
+        return func.array_to_string(
+            func.array_agg(
+                cast(func.jsonb_each_text(cls._keywords).keys(), ARRAY(String))
+            ),
+            ' '
+        )
+
+    @hybrid_property
     def text(self) -> str:
         return self.directory.configuration.extract_searchable(self.values)
 

src/onegov/fsi/views/search.py

@@ -1,8 +1,11 @@
 from onegov.core.security import Personal
 from onegov.fsi import FsiApp
-from onegov.org.models import Search
-from onegov.town6.views.search import town_search as search_view
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 from onegov.org.views.search import suggestions as suggestions_view
+from onegov.org.views.search import (suggestions_postgres as
+                                     suggestions_postgres_view)
 
 
 from typing import TYPE_CHECKING
@@ -21,9 +24,25 @@
     return search_view(self, request)
 
 
+@FsiApp.html(model=SearchPostgres, template='search.pt', permission=Personal)
+def search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'FsiRequest'
+) -> 'RenderData | Response':
+    return search_postgres_view(self, request)
+
+
 @FsiApp.json(model=Search, name='suggest', permission=Personal)
 def suggestions(
     self: Search['Base'],
     request: 'FsiRequest'
 ) -> 'JSON_ro':
     return suggestions_view(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def suggestions_postgres(
+    self: SearchPostgres['Base'],
+    request: 'FsiRequest'
+) -> 'JSON_ro':
+    return suggestions_postgres_view(self, request)

src/onegov/landsgemeinde/views/search.py

@@ -1,9 +1,8 @@
 from onegov.core.security import Public
 from onegov.landsgemeinde import LandsgemeindeApp
 from onegov.landsgemeinde.layouts import DefaultLayout
-from onegov.org.models import Search
-from onegov.org.views.search import search
-
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search, search_postgres
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -19,3 +18,12 @@
     request: 'LandsgemeindeRequest'
 ) -> 'RenderData | Response':
     return search(self, request, DefaultLayout(self, request))
+
+
+@LandsgemeindeApp.html(model=SearchPostgres, template='search.pt',
+                       permission=Public)
+def landsgemeinde_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'LandsgemeindeRequest'
+) -> 'RenderData | Response':
+    return search_postgres(self, request, DefaultLayout(self, request))

src/onegov/onboarding/app.py

@@ -2,15 +2,15 @@
 from onegov.file import DepotApp
 from onegov.onboarding.theme import OnboardingTheme
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 
 
 from typing import Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
 
-class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp):
+class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp):
 
     serve_static_files = True
 

src/onegov/onboarding/models/town_assistant.py

@@ -202,7 +202,7 @@ def add_town(
                 'org': name
             })
 
-            self.app.es_perform_reindex()
+            self.app.perform_reindex()
             self.app.send_transactional_email(
                 subject=title,
                 receivers=(user, ),

src/onegov/org/app.py

@@ -32,7 +32,7 @@
 from onegov.page import Page, PageCollection
 from onegov.pay import PayApp
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 from onegov.ticket import TicketCollection
 from onegov.ticket import TicketPermission
 from onegov.user import UserApp
@@ -59,7 +59,7 @@
     from webob import Response
 
 
-class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp,
+class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp,
              DepotApp, PayApp, FormApp, UserApp, WebsocketsApp):
 
     serve_static_files = True

src/onegov/org/layout.py

@@ -43,6 +43,7 @@
 from onegov.org.models.extensions import PersonLinkExtension
 from onegov.org.models.external_link import ExternalLinkCollection
 from onegov.org.models.form import submission_deletable
+from onegov.org.models.search import SearchPostgres
 from onegov.org.open_graph import OpenGraphMixin
 from onegov.org.theme.org_theme import user_options
 from onegov.org.utils import IMG_URLS
@@ -334,11 +335,19 @@
     @cached_property
     def search_url(self) -> str:
         """ Returns the url to the search page. """
+        # Allows using postgres search while es search remains default
+        if (self.request.path_info
+                and 'search-postgres' in self.request.path_info):
+            return self.request.class_link(SearchPostgres)
         return self.request.class_link(Search)
 
     @cached_property
     def suggestions_url(self) -> str:
         """ Returns the url to the suggestions json view. """
+        # Allows using postgres search while es search remains default
+        if (self.request.path_info
+                and 'search-postgres' in self.request.path_info):
+            return self.request.class_link(SearchPostgres, name='suggest')
         return self.request.class_link(Search, name='suggest')
 
     @cached_property

src/onegov/org/models/__init__.py

@@ -46,7 +46,7 @@
 from onegov.org.models.recipient import ResourceRecipient
 from onegov.org.models.recipient import ResourceRecipientCollection
 from onegov.org.models.resource import DaypassResource
-from onegov.org.models.search import Search
+from onegov.org.models.search import Search, SearchPostgres
 from onegov.org.models.sitecollection import SiteCollection
 from onegov.org.models.swiss_holidays import SwissHolidays
 from onegov.org.models.tan import TANAccess
@@ -101,6 +101,7 @@
     'ResourceRecipient',
     'ResourceRecipientCollection',
     'Search',
+    'SearchPostgres',
     'SiteCollection',
     'SubmissionMessage',
     'SwissHolidays',