diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e556b5df61..cecce7a1a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: - id: sass-lint files: '^src/.*\.scss' - repo: https://github.com/pre-commit/mirrors-eslint - rev: v9.9.1 + rev: v9.10.0 hooks: - id: eslint files: '^src/.*\.jsx?$' diff --git a/src/onegov/agency/views/search.py b/src/onegov/agency/views/search.py index dfdf8a9a61..32dffd04eb 100644 --- a/src/onegov/agency/views/search.py +++ b/src/onegov/agency/views/search.py @@ -1,9 +1,8 @@ from onegov.agency import AgencyApp from onegov.agency.layout import AgencySearchLayout from onegov.core.security import Public -from onegov.org.models import Search -from onegov.org.views.search import search as search_view - +from onegov.org.models import Search, SearchPostgres +from onegov.org.views.search import search as search_view, search_postgres from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -23,3 +22,14 @@ def search( if isinstance(data, dict): data['layout'] = AgencySearchLayout(self, request) return data + + +@AgencyApp.html(model=SearchPostgres, template='search.pt', permission=Public) +def agency_search_postgres( + self: SearchPostgres['Base'], + request: 'AgencyRequest' +) -> 'RenderData | Response': + data = search_postgres(self, request) + if isinstance(data, dict): + data['layout'] = AgencySearchLayout(self, request) + return data diff --git a/src/onegov/directory/models/directory_entry.py b/src/onegov/directory/models/directory_entry.py index acb6ecdc7a..91afbe45d9 100644 --- a/src/onegov/directory/models/directory_entry.py +++ b/src/onegov/directory/models/directory_entry.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import ContentMixin from onegov.core.orm.mixins import TimestampMixin @@ -6,7 +8,7 @@ from onegov.file import AssociatedFiles from onegov.gis import CoordinatesMixin from onegov.search import SearchableContent -from sqlalchemy import Column +from sqlalchemy import Column, func, cast, ARRAY, String from sqlalchemy import ForeignKey from sqlalchemy import Index from sqlalchemy import Text @@ -30,10 +32,10 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin, __tablename__ = 'directory_entries' es_properties = { - 'keywords': {'type': 'keyword'}, 'title': {'type': 'localized'}, 'lead': {'type': 'localized'}, - 'directory_id': {'type': 'keyword'}, + 'keywords': {'type': 'keyword'}, + # 'directory_id': {'type': 'keyword'}, # since the searchable text might include html, we remove it # even if there's no html -> possibly decreasing the search @@ -113,17 +115,26 @@ def external_link_visible(self) -> bool | None: def directory_name(self) -> str: return self.directory.name - @property + @hybrid_property def keywords(self) -> set[str]: return set(self._keywords.keys()) if self._keywords else set() # FIXME: asymmetric properties are not supported by mypy, switch to # a custom descriptor, if desired. - @keywords.setter + @keywords.setter # type:ignore[no-redef] def keywords(self, value: 'Collection[str] | None') -> None: self._keywords = dict.fromkeys(value, '') if value else None - @property + @keywords.expression # type:ignore[no-redef] + def keywords(cls): + return func.array_to_string( + func.array_agg( + cast(func.jsonb_each_text(cls._keywords).keys(), ARRAY(String)) + ), + ' ' + ) + + @hybrid_property def text(self) -> str: return self.directory.configuration.extract_searchable(self.values) diff --git a/src/onegov/fsi/views/search.py b/src/onegov/fsi/views/search.py index e915c4cec3..7860364182 100644 --- a/src/onegov/fsi/views/search.py +++ b/src/onegov/fsi/views/search.py @@ -1,8 +1,11 @@ from onegov.core.security import Personal from onegov.fsi import FsiApp -from onegov.org.models import Search -from onegov.town6.views.search import town_search as search_view +from onegov.org.models import Search, SearchPostgres +from onegov.org.views.search import search as search_view +from onegov.org.views.search import search_postgres as search_postgres_view from onegov.org.views.search import suggestions as suggestions_view +from onegov.org.views.search import (suggestions_postgres as + suggestions_postgres_view) from typing import TYPE_CHECKING @@ -21,9 +24,25 @@ def search( return search_view(self, request) +@FsiApp.html(model=SearchPostgres, template='search.pt', permission=Personal) +def search_postgres( + self: SearchPostgres['Base'], + request: 'FsiRequest' +) -> 'RenderData | Response': + return search_postgres_view(self, request) + + @FsiApp.json(model=Search, name='suggest', permission=Personal) def suggestions( self: Search['Base'], request: 'FsiRequest' ) -> 'JSON_ro': return suggestions_view(self, request) + + +@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal) +def suggestions_postgres( + self: SearchPostgres['Base'], + request: 'FsiRequest' +) -> 'JSON_ro': + return suggestions_postgres_view(self, request) diff --git a/src/onegov/landsgemeinde/views/search.py b/src/onegov/landsgemeinde/views/search.py index ded65c259f..acd5d05ff2 100644 --- a/src/onegov/landsgemeinde/views/search.py +++ b/src/onegov/landsgemeinde/views/search.py @@ -1,9 +1,8 @@ from onegov.core.security import Public from onegov.landsgemeinde import LandsgemeindeApp from onegov.landsgemeinde.layouts import DefaultLayout -from onegov.org.models import Search -from onegov.org.views.search import search - +from onegov.org.models import Search, SearchPostgres +from onegov.org.views.search import search, search_postgres from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -19,3 +18,12 @@ def landsgemeinde_search( request: 'LandsgemeindeRequest' ) -> 'RenderData | Response': return search(self, request, DefaultLayout(self, request)) + + +@LandsgemeindeApp.html(model=SearchPostgres, template='search.pt', + permission=Public) +def landsgemeinde_search_postgres( + self: SearchPostgres['Base'], + request: 'LandsgemeindeRequest' +) -> 'RenderData | Response': + return search_postgres(self, request, DefaultLayout(self, request)) diff --git a/src/onegov/onboarding/app.py b/src/onegov/onboarding/app.py index a460033181..ab5864d9b4 100644 --- a/src/onegov/onboarding/app.py +++ b/src/onegov/onboarding/app.py @@ -2,7 +2,7 @@ from onegov.file import DepotApp from onegov.onboarding.theme import OnboardingTheme from onegov.reservation import LibresIntegration -from onegov.search import ElasticsearchApp +from onegov.search import SearchApp from typing import Any, TYPE_CHECKING @@ -10,7 +10,7 @@ from collections.abc import Iterator -class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp): +class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp): serve_static_files = True diff --git a/src/onegov/onboarding/models/town_assistant.py b/src/onegov/onboarding/models/town_assistant.py index 87fcc47e51..3e8a9b7189 100644 --- a/src/onegov/onboarding/models/town_assistant.py +++ b/src/onegov/onboarding/models/town_assistant.py @@ -202,7 +202,7 @@ def add_town( 'org': name }) - self.app.es_perform_reindex() + self.app.perform_reindex() self.app.send_transactional_email( subject=title, receivers=(user, ), diff --git a/src/onegov/org/app.py b/src/onegov/org/app.py index d109c12afc..93f148bb8e 100644 --- a/src/onegov/org/app.py +++ b/src/onegov/org/app.py @@ -32,7 +32,7 @@ from onegov.page import Page, PageCollection from onegov.pay import PayApp from onegov.reservation import LibresIntegration -from onegov.search import ElasticsearchApp +from onegov.search import SearchApp from onegov.ticket import TicketCollection from onegov.ticket import TicketPermission from onegov.user import UserApp @@ -59,7 +59,7 @@ from webob import Response -class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp, +class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp, DepotApp, PayApp, FormApp, UserApp, WebsocketsApp): serve_static_files = True diff --git a/src/onegov/org/layout.py b/src/onegov/org/layout.py index 200279b5bc..1e3cdd11db 100644 --- a/src/onegov/org/layout.py +++ b/src/onegov/org/layout.py @@ -43,6 +43,7 @@ from onegov.org.models.extensions import PersonLinkExtension from onegov.org.models.external_link import ExternalLinkCollection from onegov.org.models.form import submission_deletable +from onegov.org.models.search import SearchPostgres from onegov.org.open_graph import OpenGraphMixin from onegov.org.theme.org_theme import user_options from onegov.org.utils import IMG_URLS @@ -334,11 +335,19 @@ def homepage_url(self) -> str: @cached_property def search_url(self) -> str: """ Returns the url to the search page. """ + # Allows using postgres search while es search remains default + if (self.request.path_info + and 'search-postgres' in self.request.path_info): + return self.request.class_link(SearchPostgres) return self.request.class_link(Search) @cached_property def suggestions_url(self) -> str: """ Returns the url to the suggestions json view. """ + # Allows using postgres search while es search remains default + if (self.request.path_info + and 'search-postgres' in self.request.path_info): + return self.request.class_link(SearchPostgres, name='suggest') return self.request.class_link(Search, name='suggest') @cached_property diff --git a/src/onegov/org/models/__init__.py b/src/onegov/org/models/__init__.py index 08d1a99e77..e96ad2f939 100644 --- a/src/onegov/org/models/__init__.py +++ b/src/onegov/org/models/__init__.py @@ -46,7 +46,7 @@ from onegov.org.models.recipient import ResourceRecipient from onegov.org.models.recipient import ResourceRecipientCollection from onegov.org.models.resource import DaypassResource -from onegov.org.models.search import Search +from onegov.org.models.search import Search, SearchPostgres from onegov.org.models.sitecollection import SiteCollection from onegov.org.models.swiss_holidays import SwissHolidays from onegov.org.models.tan import TANAccess @@ -101,6 +101,7 @@ 'ResourceRecipient', 'ResourceRecipientCollection', 'Search', + 'SearchPostgres', 'SiteCollection', 'SubmissionMessage', 'SwissHolidays', diff --git a/src/onegov/org/models/search.py b/src/onegov/org/models/search.py index 9601e2d2f5..402a144eef 100644 --- a/src/onegov/org/models/search.py +++ b/src/onegov/org/models/search.py @@ -4,25 +4,30 @@ from elasticsearch_dsl.query import MatchPhrase from elasticsearch_dsl.query import MultiMatch from functools import cached_property +from sedate import utcnow +from sqlalchemy import func +from typing import TYPE_CHECKING, Any, List + from onegov.core.collection import Pagination, _M +from onegov.core.orm import Base from onegov.event.models import Event +from onegov.search.utils import searchable_sqlalchemy_models -from typing import TYPE_CHECKING if TYPE_CHECKING: from onegov.org.request import OrgRequest + from onegov.search import Searchable from onegov.search.dsl import Hit, Response, Search as ESSearch class Search(Pagination[_M]): - results_per_page = 10 max_query_length = 100 def __init__(self, request: 'OrgRequest', query: str, page: int) -> None: super().__init__(page) self.request = request - self.query = query + self.web_search = query @cached_property def available_documents(self) -> int: @@ -35,13 +40,13 @@ def explain(self) -> bool: @property def q(self) -> str: - return self.query + return self.web_search def __eq__(self, other: object) -> bool: return ( isinstance(other, self.__class__) and self.page == other.page - and self.query == other.query + and self.web_search == other.web_search ) if TYPE_CHECKING: @@ -56,11 +61,11 @@ def page_index(self) -> int: return self.page def page_by_index(self, index: int) -> 'Search[_M]': - return Search(self.request, self.query, index) + return Search(self.request, self.web_search, index) @cached_property def batch(self) -> 'Response | None': # type:ignore[override] - if not self.query: + if not self.web_search: return None search = self.request.app.es_search_by_request( @@ -70,7 +75,7 @@ def batch(self) -> 'Response | None': # type:ignore[override] # queries need to be cut at some point to make sure we're not # pushing the elasticsearch cluster to the brink - query = self.query[:self.max_query_length] + query = self.web_search[:self.max_query_length] if query.startswith('#'): search = self.hashtag_search(search, query) @@ -95,13 +100,16 @@ def get_sort_key(event: Event) -> float: batch = self.batch.load() events = [] non_events = [] + for search_result in batch: if isinstance(search_result, Event): events.append(search_result) else: non_events.append(search_result) + if not events: return batch + sorted_events = sorted( events, key=get_sort_key @@ -147,7 +155,7 @@ def feeling_lucky(self) -> str | None: first_entry = self.batch[0].load() # XXX the default view to the event should be doing the redirect - if first_entry.__tablename__ == 'events': + if first_entry.es_type_name == 'events': return self.request.link(first_entry, 'latest') else: return self.request.link(first_entry) @@ -159,5 +167,219 @@ def subset_count(self) -> int: def suggestions(self) -> tuple[str, ...]: return tuple(self.request.app.es_suggestions_by_request( - self.request, self.query + self.request, self.web_search )) + + +def locale_mapping(locale: str) -> str: + mapping = {'de_CH': 'german', 'fr_CH': 'french', 'it_CH': 'italian', + 'rm_CH': 'english'} + return mapping.get(locale, 'english') + + +class SearchPostgres(Pagination[_M]): + """ + Implements searching in postgres db based on the gin index + """ + results_per_page = 10 + max_query_length = 100 + + def __init__(self, request: 'OrgRequest', query: str, page: int): + self.request = request + self.web_search = query + self.page = page # page index + + self.nbr_of_docs = 0 + self.nbr_of_results = 0 + + @cached_property + def available_documents(self) -> int: + if not self.nbr_of_docs: + self.load_batch_results + return self.nbr_of_docs + + @cached_property + def available_results(self) -> int: + if not self.nbr_of_results: + self.load_batch_results + return self.nbr_of_results + + @property + def q(self) -> str: + """ + Returns the user's query term from the search field of the UI + + """ + return self.web_search + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SearchPostgres): + return NotImplemented + return self.page == other.page and self.web_search == other.web_search + + def subset(self) -> 'List[Searchable] | None': # type:ignore[override] + return self.batch + + @property + def page_index(self) -> int: + return self.page + + def page_by_index(self, index: int) -> 'SearchPostgres[_M]': + return SearchPostgres(self.request, self.web_search, index) + + @cached_property + def batch(self) -> 'List[Searchable]': # type:ignore[override] + if not self.web_search: + return [] + + if self.web_search.startswith('#'): + results = self.hashtag_search() + else: + results = self.generic_search() + + return results[self.offset:self.offset + self.batch_size] + + @cached_property + def load_batch_results(self) -> list[Any]: + """ + Load search results and sort upcoming events by occurrence start date. + This methods is a wrapper around `batch.load()`, which returns the + actual search results form the query. + + """ + print('*** tschupre cached_property load_batch_results') + batch: List[Searchable] = self.batch + future_events: List[Searchable] = [] + other: List[Searchable] = [] + + for search_result in batch: + if (isinstance(search_result, Event) + and search_result.latest_occurrence + and search_result.latest_occurrence.start > utcnow()): + future_events.append(search_result) + else: + other.append(search_result) + + if not future_events: + return batch + + sorted_events = sorted( + future_events, key=lambda e: + e.latest_occurrence.start, # type:ignore[attr-defined] + reverse=True) + + return sorted_events + other + + def _create_weighted_vector( + self, + model: Any, + language: str = 'simple' + ) -> Any: + # for now weight the first field with 'A', the rest with 'B' + weighted_vector = [ + func.setweight( + func.to_tsvector( + language, + getattr(model, field, '') + ), + weight + ) + for field, weight in zip(model.es_properties.keys(), 'ABBBBBBBBBB') + if not field.startswith('es_') # TODO: rename to fts_ + ] + + # combine all weighted vectors + if weighted_vector: + combined_vector = weighted_vector[0] + for vector in weighted_vector[1:]: + combined_vector = combined_vector.op('||')(vector) + else: + combined_vector = func.to_tsvector(language, '') + + return combined_vector + + def generic_search(self) -> list['Searchable']: + doc_count = 0 + results: List[Any] = [] + language = locale_mapping(self.request.locale or 'de_CH') + ts_query = func.websearch_to_tsquery(language, self.web_search) + + for base in self.request.app.session_manager.bases: + for model in searchable_sqlalchemy_models(base): + if model.es_public or self.request.is_logged_in: # type:ignore + query = self.request.session.query(model) + + if query.count(): + doc_count += query.count() + + vector = self._create_weighted_vector(model, language) + rank_expression = func.coalesce( + func.ts_rank( + vector, + ts_query, + # 0 # normalization, ignore document length + ), 0).label('rank') + query = query.filter( + model.fts_idx.op('@@')(ts_query) + ).add_columns(rank_expression) + + results.extend(list(query.all())) + + # remove duplicates + results = list(set(results)) + + # sort by rank + results.sort(key=lambda x: x[1], reverse=True) + + # remove rank from results + results = [r[0] for r in results] + + self.nbr_of_docs = doc_count + self.nbr_of_results = len(results) + + return results + + def hashtag_search(self) -> list['Searchable']: + q = self.web_search.lstrip('#') + results = [] + + for model in searchable_sqlalchemy_models(Base): + # skip certain tables for hashtag search for better performance + if (model.es_type_name not in ['attendees', 'files', 'people', + 'tickets', 'users']): + if model.es_public or self.request.is_logged_in: # type:ignore + for doc in self.request.session.query(model).all(): + if doc.es_tags and q in doc.es_tags: + results.append(doc) + + # remove duplicates + results = list(set(results)) + + self.nbr_of_results = len(results) + return results + + def feeling_lucky(self) -> str | None: + if self.batch: + first_entry = self.batch[0] + + # XXX the default view to the event should be doing the redirect + if first_entry.es_type_name == 'events': + return self.request.link(first_entry, 'latest') + else: + return self.request.link(first_entry) + return None + + @cached_property + def subset_count(self) -> int: + return self.available_results + + def suggestions(self) -> tuple[str, ...]: + suggestions = [] + + for element in self.generic_search(): + if element.es_type_name == 'files': + continue + suggest = getattr(element, 'es_suggestion', '') + suggestions.append(suggest) + + return tuple(suggestions[:15]) diff --git a/src/onegov/org/models/ticket.py b/src/onegov/org/models/ticket.py index 4f8632d70e..132972f464 100644 --- a/src/onegov/org/models/ticket.py +++ b/src/onegov/org/models/ticket.py @@ -1,5 +1,6 @@ from functools import cached_property from markupsafe import Markup + from onegov.chat.collections import ChatCollection from onegov.core.templates import render_macro from onegov.directory import Directory, DirectoryEntry @@ -14,11 +15,11 @@ from onegov.ticket import Ticket, Handler, handlers from onegov.search.utils import extract_hashtags from purl import URL -from sqlalchemy import desc +from sqlalchemy import desc, select from sqlalchemy import func +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import object_session - from typing import Any, TYPE_CHECKING if TYPE_CHECKING: from onegov.chat.models import Chat @@ -48,6 +49,8 @@ class OrgTicketMixin: """ + _cached_extra_localized_text: str + if TYPE_CHECKING: number: Column[str] group: Column[str] @@ -67,7 +70,7 @@ def reference(self, request: 'OrgRequest') -> str: def reference_group(self, request: 'OrgRequest') -> str: return request.translate(self.group) - @cached_property + @hybrid_property def extra_localized_text(self) -> str: # extracts of attachments are currently not searchable - if they were @@ -87,8 +90,18 @@ def extra_localized_text(self) -> str: q = q.filter_by(channel_id=self.number) q = q.filter(Message.type.in_(('ticket_note', 'ticket_chat'))) q = q.with_entities(Message.text) + result = ' '.join(n.text for n in q if n.text) - return ' '.join(n.text for n in q if n.text) + return result + + @extra_localized_text.expression # type:ignore[no-redef] + def extra_localized_text(cls) -> str: + return ( + select([func.string_agg(Message.text, ' ')]) + .where(Message.channel_id == cls.number) + .where(Message.type.in_(('ticket_note', 'ticket_chat'))) + .label('extra_localized_text') + ) @property def es_tags(self) -> list[str] | None: diff --git a/src/onegov/org/path.py b/src/onegov/org/path.py index 17072d4488..87cf5747a3 100644 --- a/src/onegov/org/path.py +++ b/src/onegov/org/path.py @@ -65,7 +65,7 @@ from onegov.org.models import ResourcePersonMove from onegov.org.models import ResourceRecipient from onegov.org.models import ResourceRecipientCollection -from onegov.org.models import Search +from onegov.org.models import Search, SearchPostgres from onegov.org.models import SiteCollection from onegov.org.models import TicketNote from onegov.org.models import Topic @@ -754,6 +754,16 @@ def get_search( return Search(request, q, page) +@OrgApp.path(model=SearchPostgres, path='/search-postgres', + converters={'page': int}) +def get_search_postgres( + request: 'OrgRequest', + q: str = '', + page: int = 0 +) -> SearchPostgres[Any]: + return SearchPostgres(request, q, page) + + @OrgApp.path(model=AtoZPages, path='/a-z') def get_a_to_z(request: 'OrgRequest') -> AtoZPages: return AtoZPages(request) diff --git a/src/onegov/org/templates/search.pt b/src/onegov/org/templates/search.pt index d6cad352a7..f58aacc0e9 100644 --- a/src/onegov/org/templates/search.pt +++ b/src/onegov/org/templates/search.pt @@ -34,7 +34,7 @@ - +

${resultslabel}

diff --git a/src/onegov/org/templates/search_postgres.pt b/src/onegov/org/templates/search_postgres.pt new file mode 100644 index 0000000000..921ac7c6b5 --- /dev/null +++ b/src/onegov/org/templates/search_postgres.pt @@ -0,0 +1,70 @@ +
+ + ${title} + + +
+
+ Postgres Searching is currently unavailable due to technical + difficulties. + Please excuse the inconvenience and try again later. +
+
+ +
+
+
+ +
+ +
+ +
+
+
+
+
+
+
+
+ + +

${resultslabel}

+
+
+ +

Your postgres search returned no results.

+ +
    + +
  • + + + +
      +
    • Score: ${result.explanation['score']}
      • + +
    • + ${title}: ${result.explanation[key]['value']} +
      • +       +
    +     +
    • +     +
+ +
+
+
+ + +
\ No newline at end of file diff --git a/src/onegov/org/views/search.py b/src/onegov/org/views/search.py index 7b714ed902..0a24f9df55 100644 --- a/src/onegov/org/views/search.py +++ b/src/onegov/org/views/search.py @@ -5,6 +5,7 @@ from onegov.org.elements import Link from onegov.org.layout import DefaultLayout from onegov.org.models import Search +from onegov.org.models.search import SearchPostgres from onegov.search import SearchOfflineError from webob import exc @@ -59,9 +60,61 @@ def search( } +@OrgApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Public) +def search_postgres( + self: SearchPostgres['Base'], + request: 'OrgRequest', + layout: DefaultLayout | None = None +) -> 'RenderData | Response': + layout = layout or DefaultLayout(self, request) + assert isinstance(layout.breadcrumbs, list) + layout.breadcrumbs.append(Link(_("Search"), '#')) + + try: + searchlabel = _("Search through ${count} indexed documents", mapping={ + 'count': self.available_documents + }) + resultslabel = _("${count} Results", mapping={ + 'count': self.available_results + }) + except SearchOfflineError: + return { + 'title': _("Search Unavailable"), + 'layout': layout, + 'connection': False + } + + if 'lucky' in request.GET: + url = self.feeling_lucky() + + if url: + return morepath.redirect(url) + + return { + # TODO switch back to 'Search' once es is gone + 'title': _("Org Search Postgres"), + 'model': self, + 'layout': layout, + 'hide_search_header': True, + 'searchlabel': searchlabel, + 'resultslabel': resultslabel, + 'connection': True + } + + @OrgApp.json(model=Search, name='suggest', permission=Public) def suggestions(self: Search['Base'], request: 'OrgRequest') -> 'JSON_ro': try: return self.suggestions() except SearchOfflineError as exception: raise exc.HTTPNotFound() from exception + + +@OrgApp.json(model=SearchPostgres, name='suggest', permission=Public) +def suggestions_postgres(self: SearchPostgres['Base'], request: 'OrgRequest') \ + -> 'JSON_ro': + try: + return self.suggestions() + except SearchOfflineError as exception: + raise exc.HTTPNotFound() from exception diff --git a/src/onegov/people/models/person.py b/src/onegov/people/models/person.py index 3417c220fe..d24525788a 100644 --- a/src/onegov/people/models/person.py +++ b/src/onegov/people/models/person.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import ContentMixin from onegov.core.orm.mixins import TimestampMixin @@ -55,7 +57,7 @@ class Person(Base, ContentMixin, TimestampMixin, ORMSearchable, def es_suggestion(self) -> tuple[str, ...]: return (self.title, f'{self.first_name} {self.last_name}') - @property + @hybrid_property def title(self) -> str: """ Returns the Eastern-ordered name. """ diff --git a/src/onegov/search/__init__.py b/src/onegov/search/__init__.py index d2d5df7288..b1da536e8a 100644 --- a/src/onegov/search/__init__.py +++ b/src/onegov/search/__init__.py @@ -7,11 +7,11 @@ from onegov.search.mixins import Searchable, ORMSearchable, SearchableContent from onegov.search.dsl import Search -from onegov.search.integration import ElasticsearchApp +from onegov.search.integration import SearchApp from onegov.search.errors import SearchOfflineError __all__ = [ - 'ElasticsearchApp', + 'SearchApp', 'ORMSearchable', 'Search', 'Searchable', diff --git a/src/onegov/search/cli.py b/src/onegov/search/cli.py index 3257e929a3..6be69e260f 100644 --- a/src/onegov/search/cli.py +++ b/src/onegov/search/cli.py @@ -83,7 +83,7 @@ def run_reindex(request: 'CoreRequest', app: 'Framework') -> None: click.secho(title, underline=True) start = utcnow() - request.app.es_perform_reindex(fail) # type:ignore[attr-defined] + request.app.perform_reindex(fail) # type:ignore[attr-defined] click.secho(f"took {utcnow() - start}") diff --git a/src/onegov/search/integration.py b/src/onegov/search/integration.py index 3059369297..89f8c494ed 100644 --- a/src/onegov/search/integration.py +++ b/src/onegov/search/integration.py @@ -23,7 +23,8 @@ from urllib3.exceptions import HTTPError -from typing import Any, Literal, TYPE_CHECKING +from typing import Any, Literal, TYPE_CHECKING, List + if TYPE_CHECKING: from collections.abc import Callable, Iterable from datetime import datetime @@ -109,9 +110,8 @@ def is_5xx_error(error: TransportError) -> bool: return False -# TODO rename to SearchApp -class ElasticsearchApp(morepath.App): - """ Provides elasticsearch integration for +class SearchApp(morepath.App): + """ Provides elasticsearch and postgres integration for :class:`onegov.core.framework.Framework` based applications. The application must be connected to a database. @@ -178,6 +178,8 @@ def configure_search(self, **cfg: Any) -> None: - fr """ + # TODO: set default to False once fully switched to psql (or remove + # es stuff entirely) if not cfg.get('enable_elasticsearch', True): self.es_client = None return @@ -419,7 +421,7 @@ def get_searchable_models(self) -> list[type['Searchable']]: for model in searchable_sqlalchemy_models(base) ] - def es_perform_reindex(self, fail: bool = False) -> None: + def perform_reindex(self, fail: bool = False) -> None: """ Re-indexes all content. This is a heavy operation and should be run with consideration. @@ -428,7 +430,7 @@ def es_perform_reindex(self, fail: bool = False) -> None: """ # prevent tables get re-indexed twice - index_done = [] + index_done: List[str] = [] schema = self.schema index_log.info(f'Indexing schema {schema}..') @@ -470,12 +472,13 @@ def reindex_model(model: type['Base']) -> None: session.invalidate() session.bind.dispose() - models = self.get_searchable_models() - index_log.info(f'Number of models to be indexed: {len(models)}') - with ThreadPoolExecutor() as executor: results = executor.map( - reindex_model, (model for model in models) + reindex_model, ( + model + for base in self.session_manager.bases + for model in searchable_sqlalchemy_models(base) + ) ) if fail: print(tuple(results)) @@ -484,14 +487,14 @@ def reindex_model(model: type['Base']) -> None: self.psql_indexer.bulk_process() -@ElasticsearchApp.tween_factory(over=transaction_tween_factory) +@SearchApp.tween_factory(over=transaction_tween_factory) def process_indexer_tween_factory( - app: ElasticsearchApp, + app: SearchApp, handler: 'Callable[[CoreRequest], Response]' ) -> 'Callable[[CoreRequest], Response]': def process_indexer_tween(request: 'CoreRequest') -> 'Response': - app: ElasticsearchApp = request.app # type:ignore[assignment] + app: SearchApp = request.app # type:ignore[assignment] if not app.es_client: return handler(request) diff --git a/src/onegov/search/mixins.py b/src/onegov/search/mixins.py index 02e64668cd..2dd67eef25 100644 --- a/src/onegov/search/mixins.py +++ b/src/onegov/search/mixins.py @@ -52,7 +52,7 @@ def es_type_name(self): """ TEXT_SEARCH_COLUMN_NAME = 'fts_idx' - TEXT_SEARCH_DATA_COLUMN_NAME = 'fts_idx_data' + # TEXT_SEARCH_DATA_COLUMN_NAME = 'fts_idx_data' if TYPE_CHECKING: # NOTE: This doesn't really have a Python representation, unless diff --git a/src/onegov/ticket/model.py b/src/onegov/ticket/model.py index 88333e6ca4..5e4ba60835 100644 --- a/src/onegov/ticket/model.py +++ b/src/onegov/ticket/model.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import TimestampMixin from onegov.core.orm.types import JSON, UUID @@ -8,7 +10,7 @@ from onegov.user import User from onegov.user import UserGroup from sedate import utcnow -from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, Text +from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, Text, text from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import backref, deferred, relationship from uuid import uuid4 @@ -152,13 +154,17 @@ def created(cls) -> 'Column[datetime]': 'extra_localized_text': {'type': 'localized'} } - @property - def extra_localized_text(self) -> str | None: + @hybrid_property + def extra_localized_text(self) -> str: """ Maybe used by child-classes to return localized extra data that should be indexed as well. """ - return None + return '' + + @extra_localized_text.expression # type:ignore[no-redef] + def extra_localized_text(cls) -> str: + return text('') @property def es_suggestion(self) -> list[str]: @@ -167,14 +173,14 @@ def es_suggestion(self) -> list[str]: self.number.replace('-', '') ] - @property + @hybrid_property def ticket_email(self) -> str | None: if self.handler.deleted: return self.snapshot.get('email') else: return self.handler.email - @property + @hybrid_property def ticket_data(self) -> 'Sequence[str] | None': if self.handler.deleted: return self.snapshot.get('summary') diff --git a/src/onegov/town6/templates/macros.pt b/src/onegov/town6/templates/macros.pt index e8f4332058..a9e82e588d 100644 --- a/src/onegov/town6/templates/macros.pt +++ b/src/onegov/town6/templates/macros.pt @@ -1813,9 +1813,9 @@ ${result.number}
    -
  • ${translate(result.group)}
    • -
  • -
  • +
  • ${translate(result.group)}
    • +
  • +
diff --git a/src/onegov/town6/templates/search.pt b/src/onegov/town6/templates/search.pt index d1b5291270..f74f9ec927 100644 --- a/src/onegov/town6/templates/search.pt +++ b/src/onegov/town6/templates/search.pt @@ -15,7 +15,7 @@
- +
- +

${resultslabel}

diff --git a/src/onegov/town6/templates/search_postgres.pt b/src/onegov/town6/templates/search_postgres.pt new file mode 100644 index 0000000000..2b6e190e31 --- /dev/null +++ b/src/onegov/town6/templates/search_postgres.pt @@ -0,0 +1,68 @@ +
+ + ${title} + + +
+
+ Postgres Searching is currently unavailable due to technical + difficulties. + Please excuse the inconvenience and try again later. +
+
+ +
+
+ + +
+ +
+ +
+
+
+
+
+ +
+
+ + +

${resultslabel}

+
+
+ +

Your postgres search returned no results.

+ +
    + +
  • + + + +
      +
    • Score: ${result.explanation['score']}
      • + +
    • + ${title}: ${result.explanation[key]['value']} +
      • +       +
    +     +
    • +     +
+ +
+
+
+ + +
\ No newline at end of file diff --git a/src/onegov/town6/views/search.py b/src/onegov/town6/views/search.py index db31ec7a48..c035884d9a 100644 --- a/src/onegov/town6/views/search.py +++ b/src/onegov/town6/views/search.py @@ -1,10 +1,10 @@ from onegov.core.security import Public -from onegov.org.views.search import search -from onegov.town6 import TownApp from onegov.org.models import Search +from onegov.org.models.search import SearchPostgres +from onegov.org.views.search import search, search_postgres +from onegov.town6 import TownApp from onegov.town6.layout import DefaultLayout - from typing import TYPE_CHECKING if TYPE_CHECKING: from onegov.core.orm import Base @@ -19,3 +19,12 @@ def town_search( request: 'TownRequest' ) -> 'RenderData | Response': return search(self, request, DefaultLayout(self, request)) + + +@TownApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Public) +def town_search_postgres( + self: SearchPostgres['Base'], + request: 'TownRequest' +) -> 'RenderData | Response': + return search_postgres(self, request, DefaultLayout(self, request)) diff --git a/src/onegov/user/models/user.py b/src/onegov/user/models/user.py index d9e7d3d300..53627a1f34 100644 --- a/src/onegov/user/models/user.py +++ b/src/onegov/user/models/user.py @@ -59,7 +59,7 @@ class User(Base, TimestampMixin, ORMSearchable): def es_suggestion(self) -> tuple[str, str]: return (self.realname or self.username, self.username) - @property + @hybrid_property def userprofile(self) -> list[str]: if not self.data: return [] diff --git a/tests/onegov/search/conftest.py b/tests/onegov/search/conftest.py index da452eb3f7..f970b20325 100644 --- a/tests/onegov/search/conftest.py +++ b/tests/onegov/search/conftest.py @@ -3,7 +3,7 @@ from yaml import dump from onegov.core import Framework -from onegov.search import ElasticsearchApp +from onegov.search import SearchApp from tests.shared.utils import create_app @@ -46,7 +46,7 @@ def cfg_path(postgres_dsn, session_manager, temporary_directory, redis_url): return cfg_path -class TestApp(Framework, ElasticsearchApp): +class TestApp(Framework, SearchApp): pass diff --git a/tests/onegov/search/test_integration.py b/tests/onegov/search/test_integration.py index 069519a1ae..2a1b0ec0e1 100644 --- a/tests/onegov/search/test_integration.py +++ b/tests/onegov/search/test_integration.py @@ -8,7 +8,7 @@ from onegov.core import Framework from onegov.core.orm.mixins import TimestampMixin from onegov.core.utils import scan_morepath_modules -from onegov.search import ElasticsearchApp, ORMSearchable +from onegov.search import SearchApp, ORMSearchable from sqlalchemy import Boolean, Column, Integer, Text from sqlalchemy.ext.declarative import declarative_base from webtest import TestApp as Client @@ -17,7 +17,7 @@ def test_app_integration(es_url): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass app = App() @@ -34,7 +34,7 @@ class App(Framework, ElasticsearchApp): def test_search_query(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -154,7 +154,7 @@ def es_language(self): def test_orm_integration(es_url, postgres_dsn, redis_url): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -299,7 +299,7 @@ def view_delete_document(self, request): def test_alternate_id_property(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -363,7 +363,7 @@ def es_suggestion(self): def test_orm_polymorphic(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -451,7 +451,7 @@ def update(): def test_orm_polymorphic_sublcass_only(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -509,7 +509,7 @@ def es_suggestion(self): def test_suggestions(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -630,7 +630,7 @@ def es_suggestion(self): def test_language_detection(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -686,7 +686,7 @@ class Document(Base, ORMSearchable): def test_language_update(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -742,7 +742,7 @@ class Document(Base, ORMSearchable): def test_date_decay(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base()