diff --git a/src/onegov/activity/models/attendee.py b/src/onegov/activity/models/attendee.py
index efc3a943d2..1538a8c7ec 100644
--- a/src/onegov/activity/models/attendee.py
+++ b/src/onegov/activity/models/attendee.py
@@ -1,4 +1,5 @@
from datetime import date
+
from onegov.activity.models.booking import Booking
from onegov.core.orm import Base
from onegov.core.orm.mixins import TimestampMixin
@@ -11,7 +12,6 @@
from sqlalchemy import Date
from sqlalchemy import Float
from sqlalchemy import ForeignKey
-from sqlalchemy import Index
from sqlalchemy import Integer
from sqlalchemy import Numeric
from sqlalchemy import Text
@@ -43,6 +43,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable):
}
es_public = False
+ @property
+ def search_score(self):
+ return 3
+
@property
def es_suggestion(self):
return self.name
@@ -179,7 +183,3 @@ def happiness(cls, period_id):
order_by='Booking.created',
backref='attendee'
)
-
- __table_args__ = (
- Index('unique_child_name', 'username', 'name', unique=True),
- )
diff --git a/src/onegov/agency/models/person.py b/src/onegov/agency/models/person.py
index 6a62a2499b..2a56f24f8c 100644
--- a/src/onegov/agency/models/person.py
+++ b/src/onegov/agency/models/person.py
@@ -21,8 +21,6 @@ def es_public(self):
'title': {'type': 'text'},
'function': {'type': 'localized'},
'email': {'type': 'text'},
- 'phone_internal': {'type': 'text'},
- 'phone_es': {'type': 'text'}
}
@property
diff --git a/src/onegov/agency/views/search.py b/src/onegov/agency/views/search.py
index ff197aaaf1..740159649b 100644
--- a/src/onegov/agency/views/search.py
+++ b/src/onegov/agency/views/search.py
@@ -1,13 +1,23 @@
from onegov.agency import AgencyApp
from onegov.agency.layout import AgencySearchLayout
from onegov.core.security import Public
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
@AgencyApp.html(model=Search, template='search.pt', permission=Public)
-def search(self, request):
+def agency_search(self, request):
data = search_view(self, request)
if isinstance(data, dict):
data['layout'] = AgencySearchLayout(self, request)
return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+ permission=Public)
+def agency_search_postgres(self, request):
+ data = search_postgres_view(self, request)
+ if isinstance(data, dict):
+ data['layout'] = AgencySearchLayout(self, request)
+ return data
diff --git a/src/onegov/core/upgrade.py b/src/onegov/core/upgrade.py
index 1b409fafdc..eff83c4ebe 100644
--- a/src/onegov/core/upgrade.py
+++ b/src/onegov/core/upgrade.py
@@ -480,6 +480,12 @@ def has_column(self, table: str, column: str) -> bool:
table, schema=self.schema
)}
+ def has_index(self, table: str, index: str) -> bool:
+ inspector = Inspector(self.operations_connection)
+ return index in {i['name'] for i in inspector.get_indexes(
+ table, schema=self.schema
+ )}
+
def has_enum(self, enum: str) -> bool:
return self.session.execute(f"""
SELECT EXISTS (
diff --git a/src/onegov/directory/models/directory.py b/src/onegov/directory/models/directory.py
index 6c1c66d0c3..c03f3a224f 100644
--- a/src/onegov/directory/models/directory.py
+++ b/src/onegov/directory/models/directory.py
@@ -17,10 +17,8 @@
from onegov.file.utils import as_fileintent
from onegov.form import flatten_fieldsets, parse_formcode, parse_form
from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, Text, Integer
from sqlalchemy import func, exists, and_
-from sqlalchemy import Integer
-from sqlalchemy import Text
from sqlalchemy.orm import object_session
from sqlalchemy.orm import relationship
from sqlalchemy.orm.attributes import InstrumentedAttribute
@@ -103,6 +101,10 @@ def count(self):
backref='directory'
)
+ @property
+ def search_score(self):
+ return 7
+
@property
def entry_cls_name(self):
return 'DirectoryEntry'
diff --git a/src/onegov/directory/models/directory_entry.py b/src/onegov/directory/models/directory_entry.py
index 4f92fe3828..49d0fb9ce2 100644
--- a/src/onegov/directory/models/directory_entry.py
+++ b/src/onegov/directory/models/directory_entry.py
@@ -6,12 +6,13 @@
from onegov.file import AssociatedFiles
from onegov.gis import CoordinatesMixin
from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, cast
from sqlalchemy import ForeignKey
from sqlalchemy import Index
from sqlalchemy import Text
from sqlalchemy.dialects.postgresql import HSTORE
from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.ext.hybrid import hybrid_property
from uuid import uuid4
@@ -25,7 +26,7 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
'keywords': {'type': 'keyword'},
'title': {'type': 'localized'},
'lead': {'type': 'localized'},
- 'directory_id': {'type': 'keyword'},
+ '_directory_id': {'type': 'keyword'},
# since the searchable text might include html, we remove it
# even if there's no html -> possibly decreasing the search
@@ -86,6 +87,10 @@ def external_link_title(self):
def external_link_visible(self):
return self.directory.configuration.link_visible
+ @hybrid_property
+ def _directory_id(self):
+ return cast(self.directory_id, Text)
+
@property
def directory_name(self):
return self.directory.name
@@ -98,7 +103,7 @@ def keywords(self):
def keywords(self, value):
self._keywords = {k: '' for k in value} if value else None
- @property
+ @hybrid_property
def text(self):
return self.directory.configuration.extract_searchable(self.values)
diff --git a/src/onegov/event/models/event.py b/src/onegov/event/models/event.py
index f69e9ef089..6d4107050a 100644
--- a/src/onegov/event/models/event.py
+++ b/src/onegov/event/models/event.py
@@ -6,6 +6,7 @@
from icalendar import Calendar as vCalendar
from icalendar import Event as vEvent
from icalendar import vRecur
+from sqlalchemy.ext.hybrid import hybrid_property
from onegov.core.orm import Base
from onegov.core.orm.abstract import associated
@@ -109,6 +110,10 @@ class Event(Base, OccurrenceMixin, TimestampMixin, SearchableContent,
EventFile, 'pdf', 'one-to-one', uselist=False, backref_suffix='pdf'
)
+ @property
+ def search_score(self):
+ return 1
+
def set_image(self, content, filename=None):
self.set_blob('image', content, filename)
@@ -148,9 +153,16 @@ def set_blob(self, blob, content, filename=None):
'description': {'type': 'localized'},
'location': {'type': 'localized'},
'organizer': {'type': 'localized'},
- 'filter_keywords': {'type': 'keyword'}
}
+ @hybrid_property
+ def description(self): # noqa: F811
+ return self.content['description'].astext
+
+ @hybrid_property
+ def organizer(self): # noqa: F811
+ return self.content['organizer'].astext
+
@property
def es_public(self):
return self.state == 'published'
diff --git a/src/onegov/feriennet/models/activity.py b/src/onegov/feriennet/models/activity.py
index 4665613c14..42bc2e6bcd 100644
--- a/src/onegov/feriennet/models/activity.py
+++ b/src/onegov/feriennet/models/activity.py
@@ -1,4 +1,7 @@
from functools import cached_property
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.activity import Activity, ActivityCollection, Occasion
from onegov.activity import PublicationRequestCollection
from onegov.activity.models import DAYS
@@ -24,6 +27,10 @@ class VacationActivity(Activity, CoordinatesExtension, SearchableContent):
'organiser': {'type': 'text'}
}
+ @property
+ def search_score(self):
+ return 1
+
@property
def es_public(self):
return self.state == 'accepted'
@@ -32,7 +39,7 @@ def es_public(self):
def es_skip(self):
return self.state == 'preview'
- @property
+ @hybrid_property
def organiser(self):
organiser = [
self.user.username,
diff --git a/src/onegov/feriennet/templates/mail_booking_accepted.pt b/src/onegov/feriennet/templates/mail_booking_accepted.pt
index 46ada5cd6e..ebd8c3c1c2 100644
--- a/src/onegov/feriennet/templates/mail_booking_accepted.pt
+++ b/src/onegov/feriennet/templates/mail_booking_accepted.pt
@@ -16,7 +16,7 @@
- Best regards
+ Best regards
diff --git a/src/onegov/feriennet/templates/mail_booking_canceled.pt b/src/onegov/feriennet/templates/mail_booking_canceled.pt
index 5b272ec3fe..1e35c9543a 100644
--- a/src/onegov/feriennet/templates/mail_booking_canceled.pt
+++ b/src/onegov/feriennet/templates/mail_booking_canceled.pt
@@ -15,7 +15,7 @@
- Best regards
+ Best regards
diff --git a/src/onegov/file/models/file.py b/src/onegov/file/models/file.py
index 6806ce330f..3ce25ca429 100644
--- a/src/onegov/file/models/file.py
+++ b/src/onegov/file/models/file.py
@@ -6,6 +6,7 @@
from contextlib import contextmanager
from collections import defaultdict
from depot.fields.sqlalchemy import UploadedFileField as UploadedFileFieldBase
+
from onegov.core.crypto import random_token
from onegov.core.orm import Base
from onegov.core.orm.abstract import Associable
@@ -262,6 +263,10 @@ class File(Base, Associable, TimestampMixin):
Index('files_by_type_and_order', 'type', 'order'),
)
+ @property
+ def search_score(self) -> int:
+ return 10
+
@hybrid_property
def signature_timestamp(self) -> 'datetime | None':
if self.signed:
diff --git a/src/onegov/file/models/fileset.py b/src/onegov/file/models/fileset.py
index 5ae1a1c37e..51cccbbd2c 100644
--- a/src/onegov/file/models/fileset.py
+++ b/src/onegov/file/models/fileset.py
@@ -6,6 +6,7 @@
from typing import TYPE_CHECKING
+
if TYPE_CHECKING:
from .file import File
diff --git a/src/onegov/form/models/definition.py b/src/onegov/form/models/definition.py
index 26426a04cb..85673fb2c5 100644
--- a/src/onegov/form/models/definition.py
+++ b/src/onegov/form/models/definition.py
@@ -140,6 +140,9 @@ class FormDefinition(Base, ContentMixin, TimestampMixin, Extendable):
}
@property
+ def search_score(self) -> int:
+ return 7
+
def form_class(self) -> Type['Form']:
""" Parses the form definition and returns a form class. """
diff --git a/src/onegov/fsi/models/course.py b/src/onegov/fsi/models/course.py
index 5c79d353da..04c6957469 100644
--- a/src/onegov/fsi/models/course.py
+++ b/src/onegov/fsi/models/course.py
@@ -1,4 +1,5 @@
from arrow import utcnow
+
from onegov.core.html import html_to_text
from onegov.core.orm import Base
from onegov.core.orm.types import UUID
@@ -32,6 +33,10 @@ class Course(Base, ORMSearchable):
# hides the course in the collection for non-admins
hidden_from_public = Column(Boolean, nullable=False, default=False)
+ @property
+ def search_score(self):
+ return 2
+
@property
def title(self):
return self.name
diff --git a/src/onegov/fsi/models/course_attendee.py b/src/onegov/fsi/models/course_attendee.py
index 5a243ed852..ceaee8127c 100644
--- a/src/onegov/fsi/models/course_attendee.py
+++ b/src/onegov/fsi/models/course_attendee.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.types import UUID, JSON
from sqlalchemy import Boolean
@@ -107,7 +109,7 @@ def __str__(self):
cascade='all, delete-orphan'
)
- @property
+ @hybrid_property
def title(self):
return ' '.join((
p for p in (
@@ -131,7 +133,7 @@ def role(self):
return 'member'
return self.user.role
- @property
+ @hybrid_property
def email(self):
"""Needs a switch for external users"""
if not self.user_id:
diff --git a/src/onegov/fsi/models/course_event.py b/src/onegov/fsi/models/course_event.py
index cbe27dd4d4..706302a419 100644
--- a/src/onegov/fsi/models/course_event.py
+++ b/src/onegov/fsi/models/course_event.py
@@ -22,13 +22,15 @@
from onegov.fsi.models.course_subscription import subscription_table
from onegov.search import ORMSearchable
-
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .course import Course
from .course_notification_template import (
- CancellationTemplate, CourseNotificationTemplate, InfoTemplate,
- ReminderTemplate, SubscriptionTemplate
+ CancellationTemplate,
+ CourseNotificationTemplate,
+ InfoTemplate,
+ ReminderTemplate,
+ SubscriptionTemplate
)
COURSE_EVENT_STATUSES = ('created', 'confirmed', 'canceled', 'planned')
@@ -83,7 +85,7 @@ def es_public(self):
def title(self):
return str(self)
- @property
+ @hybrid_property
def name(self):
return self.course.name
@@ -95,7 +97,7 @@ def lead(self):
f'{self.presenter_company}'
)
- @property
+ @hybrid_property
def description(self):
return self.course.description
diff --git a/src/onegov/fsi/views/search.py b/src/onegov/fsi/views/search.py
index 22a659a330..b1b3c8980f 100644
--- a/src/onegov/fsi/views/search.py
+++ b/src/onegov/fsi/views/search.py
@@ -1,8 +1,11 @@
from onegov.core.security import Personal
from onegov.fsi import FsiApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
from onegov.org.views.search import suggestions as suggestions_view
+from onegov.org.views.search import suggestions_postgres as \
+ suggestions_postgres_view
@FsiApp.html(model=Search, template='search.pt', permission=Personal)
@@ -10,6 +13,17 @@ def search(self, request):
return search_view(self, request)
+@FsiApp.html(model=SearchPostgres, template='search_postgres.pt',
+ permission=Personal)
+def search_postgres(self, request):
+ return search_postgres_view(self, request)
+
+
@FsiApp.json(model=Search, name='suggest', permission=Personal)
def suggestions(self, request):
return suggestions_view(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def suggestions_postgres(self, request):
+ return suggestions_postgres_view(self, request)
diff --git a/src/onegov/landsgemeinde/models/agenda.py b/src/onegov/landsgemeinde/models/agenda.py
index e75e018dad..4f6f26f123 100644
--- a/src/onegov/landsgemeinde/models/agenda.py
+++ b/src/onegov/landsgemeinde/models/agenda.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.mixins import content_property
from onegov.core.orm.mixins import ContentMixin
@@ -105,6 +107,18 @@ class AgendaItem(
last_modified = Column(UTCDateTime)
+ @hybrid_property
+ def overview(self): # noqa: F811
+ return self.content['overview'].astext
+
+ @hybrid_property
+ def text(self): # noqa: F811
+ return self.content['text'].astext
+
+ @hybrid_property
+ def resolution(self): # noqa: F811
+ return self.content['resolution'].astext
+
def stamp(self):
self.last_modified = self.timestamp()
diff --git a/src/onegov/landsgemeinde/models/assembly.py b/src/onegov/landsgemeinde/models/assembly.py
index ed4d4367a1..e4124be56d 100644
--- a/src/onegov/landsgemeinde/models/assembly.py
+++ b/src/onegov/landsgemeinde/models/assembly.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.mixins import content_property
from onegov.core.orm.mixins import ContentMixin
@@ -94,5 +96,9 @@ def es_suggestion(self):
last_modified = Column(UTCDateTime)
+ @hybrid_property
+ def overview(self): # noqa: F811
+ return self.content['overview'].astext
+
def stamp(self):
self.last_modified = self.timestamp()
diff --git a/src/onegov/landsgemeinde/models/votum.py b/src/onegov/landsgemeinde/models/votum.py
index 96ab78e552..b5c3746de2 100644
--- a/src/onegov/landsgemeinde/models/votum.py
+++ b/src/onegov/landsgemeinde/models/votum.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.mixins import content_property
from onegov.core.orm.mixins import ContentMixin
@@ -92,6 +94,18 @@ def es_suggestion(self):
nullable=False
)
+ @hybrid_property
+ def text(self): # noqa: F811
+ return self.content['text'].astext
+
+ @hybrid_property
+ def motion(self): # noqa: F811
+ return self.content['motion'].astext
+
+ @hybrid_property
+ def statement_of_reasons(self): # noqa: F811
+ return self.content['statement_of_reasons'].astext
+
@property
def date(self):
return self.agenda_item.date
diff --git a/src/onegov/landsgemeinde/views/search.py b/src/onegov/landsgemeinde/views/search.py
index 04d3e17b1a..ef759e8166 100644
--- a/src/onegov/landsgemeinde/views/search.py
+++ b/src/onegov/landsgemeinde/views/search.py
@@ -7,4 +7,5 @@
@LandsgemeindeApp.html(model=Search, template='search.pt', permission=Public)
def landsgemeinde_search(self, request):
+ # TODO: switch to postgres search
return search(self, request, DefaultLayout(self, request))
diff --git a/src/onegov/newsletter/models.py b/src/onegov/newsletter/models.py
index 44500057d0..9e93a7a87b 100644
--- a/src/onegov/newsletter/models.py
+++ b/src/onegov/newsletter/models.py
@@ -95,6 +95,9 @@ def validate_name(self, key: str, name: str) -> str:
back_populates='newsletters')
@property
+ def search_score(self) -> int:
+ return 6
+
def open_recipients(self) -> tuple['Recipient', ...]:
received = select([newsletter_recipients.c.recipient_id]).where(
newsletter_recipients.c.newsletter_id == self.name)
diff --git a/src/onegov/onboarding/app.py b/src/onegov/onboarding/app.py
index a460033181..ab5864d9b4 100644
--- a/src/onegov/onboarding/app.py
+++ b/src/onegov/onboarding/app.py
@@ -2,7 +2,7 @@
from onegov.file import DepotApp
from onegov.onboarding.theme import OnboardingTheme
from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
from typing import Any, TYPE_CHECKING
@@ -10,7 +10,7 @@
from collections.abc import Iterator
-class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp):
+class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp):
serve_static_files = True
diff --git a/src/onegov/org/app.py b/src/onegov/org/app.py
index 120b5d71e4..450c10efb2 100644
--- a/src/onegov/org/app.py
+++ b/src/onegov/org/app.py
@@ -29,7 +29,7 @@
from onegov.page import Page, PageCollection
from onegov.pay import PayApp
from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
from onegov.ticket import TicketCollection
from onegov.ticket import TicketPermission
from onegov.user import UserApp
@@ -47,7 +47,7 @@
from reg.dispatch import _KeyLookup
-class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp,
+class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp,
DepotApp, PayApp, FormApp, UserApp, WebsocketsApp):
serve_static_files = True
diff --git a/src/onegov/org/layout.py b/src/onegov/org/layout.py
index 9943c2cd9a..586a2290d3 100644
--- a/src/onegov/org/layout.py
+++ b/src/onegov/org/layout.py
@@ -26,7 +26,7 @@
from onegov.org import _
from onegov.org import utils
from onegov.org.exports.base import OrgExport
-from onegov.org.models import ExportCollection, Editor
+from onegov.org.models import ExportCollection, Editor, SearchPostgres
from onegov.org.models import GeneralFileCollection
from onegov.org.models import ImageFile
from onegov.org.models import ImageFileCollection
@@ -290,11 +290,16 @@ def homepage_url(self):
@cached_property
def search_url(self):
""" Returns the url to the search page. """
+ if 'search_postgres' in self.request.path_info:
+ return self.request.link(SearchPostgres(self.request, None, None))
return self.request.link(Search(self.request, None, None))
@cached_property
def suggestions_url(self):
""" Returns the url to the suggestions json view. """
+ if 'search_postgres' in self.request.path_info:
+ return self.request.link(SearchPostgres(self.request, None,
+ None), 'suggest')
return self.request.link(Search(self.request, None, None), 'suggest')
@cached_property
diff --git a/src/onegov/org/models/__init__.py b/src/onegov/org/models/__init__.py
index 32ac6f3b55..a5102c6073 100644
--- a/src/onegov/org/models/__init__.py
+++ b/src/onegov/org/models/__init__.py
@@ -46,7 +46,7 @@
from onegov.org.models.recipient import ResourceRecipient
from onegov.org.models.recipient import ResourceRecipientCollection
from onegov.org.models.resource import DaypassResource
-from onegov.org.models.search import Search
+from onegov.org.models.search import Search, SearchPostgres
from onegov.org.models.sitecollection import SiteCollection
from onegov.org.models.swiss_holidays import SwissHolidays
from onegov.org.models.tan import TAN
@@ -103,6 +103,7 @@
'ResourceRecipient',
'ResourceRecipientCollection',
'Search',
+ 'SearchPostgres',
'SiteCollection',
'SubmissionMessage',
'SwissHolidays',
diff --git a/src/onegov/org/models/external_link.py b/src/onegov/org/models/external_link.py
index 600e1a5276..f3f06a428b 100644
--- a/src/onegov/org/models/external_link.py
+++ b/src/onegov/org/models/external_link.py
@@ -1,5 +1,7 @@
from uuid import uuid4
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.collection import GenericCollection
from onegov.core.orm import Base
from onegov.core.orm.mixins import ContentMixin, \
@@ -48,10 +50,18 @@ class ExternalLink(Base, ContentMixin, TimestampMixin, AccessExtension,
lead = meta_property()
+ @property
+ def search_score(self):
+ return 8
+
@observes('title')
def title_observer(self, title):
self.order = normalize_for_url(title)
+ @hybrid_property
+ def lead(self): # noqa: F811
+ return self.meta['lead'].astext
+
class ExternalLinkCollection(GenericCollection):
diff --git a/src/onegov/org/models/file.py b/src/onegov/org/models/file.py
index 49f2ef3d65..dbe729863e 100644
--- a/src/onegov/org/models/file.py
+++ b/src/onegov/org/models/file.py
@@ -7,6 +7,9 @@
from dateutil.relativedelta import relativedelta
from functools import cached_property
from itertools import chain, groupby
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import as_selectable
from onegov.core.orm.mixins import meta_property
from onegov.file import File, FileSet, FileCollection, FileSetCollection
@@ -153,6 +156,14 @@ def es_suggestions(self):
show_images_on_homepage = meta_property()
+ @hybrid_property
+ def lead(self): # noqa: F811
+ return self.meta['lead'].astext
+
+ @hybrid_property
+ def view(self): # noqa: F811
+ return self.meta['view'].astext
+
class ImageSetCollection(FileSetCollection):
diff --git a/src/onegov/org/models/form.py b/src/onegov/org/models/form.py
index 24c3a67e3a..31aac78788 100644
--- a/src/onegov/org/models/form.py
+++ b/src/onegov/org/models/form.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.form.models import FormDefinition
from onegov.org.models.extensions import AccessExtension
from onegov.org.models.extensions import ContactExtension
@@ -21,6 +23,14 @@ class BuiltinFormDefinition(FormDefinition, AccessExtension,
def extensions(self):
return tuple(set(super().extensions + ['honeypot']))
+ @hybrid_property
+ def lead(self):
+ return self.meta['lead'].astext
+
+ @hybrid_property
+ def text(self):
+ return self.content['text'].astext
+
class CustomFormDefinition(FormDefinition, AccessExtension,
ContactExtension, PersonLinkExtension,
diff --git a/src/onegov/org/models/page.py b/src/onegov/org/models/page.py
index 4d7ab1e90a..05b9dfbcab 100644
--- a/src/onegov/org/models/page.py
+++ b/src/onegov/org/models/page.py
@@ -1,4 +1,5 @@
from datetime import datetime
+
from onegov.core.orm.mixins import (
content_property, dict_property, meta_property)
from onegov.file import MultiAssociatedFiles
@@ -21,6 +22,7 @@
from sedate import replace_timezone
from sqlalchemy import desc, func, or_, and_
from sqlalchemy.dialects.postgresql import array, JSON
+from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import undefer, object_session
from sqlalchemy_utils import observes
@@ -42,6 +44,30 @@ class Topic(Page, TraitInfo, SearchableContent, AccessExtension,
# Show the lead on topics page
lead_when_child = content_property(default=True)
+ @hybrid_property
+ def lead(self): # noqa: F811
+ return self.content['lead']
+
+ @lead.expression
+ def lead(cls):
+ return cls.content['lead'].astext
+
+ @hybrid_property
+ def text(self): # noqa: F811
+ return self.content['text']
+
+ @text.expression
+ def text(cls):
+ return cls.content['text'].astext
+
+ @hybrid_property
+ def url(self): # noqa: F811
+ return self.content['url']
+
+ @url.expression
+ def url(cls):
+ return cls.content['url'].astext
+
@property
def es_skip(self):
return self.meta.get('trait') == 'link' # do not index links
diff --git a/src/onegov/org/models/search.py b/src/onegov/org/models/search.py
index 7ff35b316d..fbd25b4a2d 100644
--- a/src/onegov/org/models/search.py
+++ b/src/onegov/org/models/search.py
@@ -1,11 +1,17 @@
+from functools import cached_property
+from operator import attrgetter
+
from elasticsearch_dsl.function import SF
from elasticsearch_dsl.query import FunctionScore
from elasticsearch_dsl.query import Match
from elasticsearch_dsl.query import MatchPhrase
from elasticsearch_dsl.query import MultiMatch
-from functools import cached_property
+from sqlalchemy import func
+
from onegov.core.collection import Pagination
+from onegov.core.orm import Base
from onegov.event.models import Event
+from onegov.search.utils import searchable_sqlalchemy_models
class Search(Pagination):
@@ -133,3 +139,149 @@ def suggestions(self):
return tuple(self.request.app.es_suggestions_by_request(
self.request, self.query
))
+
+
+def locale_mapping(locale):
+ mapping = {'de_CH': 'german', 'fr_CH': 'french', 'it_CH': 'italian',
+ 'rm_CH': 'english'}
+ return mapping.get(locale, 'english')
+
+
+class SearchPostgres(Pagination):
+ """
+ Implements searching in postgres db based on the gin index
+ """
+ results_per_page = 10
+ max_query_length = 100
+
+ def __init__(self, request, query, page):
+ self.request = request
+ self.query = query
+ self.page = page # page index
+
+ self.nbr_of_docs = 0
+ self.nbr_of_results = 0
+
+ @cached_property
+ def available_documents(self):
+ if not self.nbr_of_docs:
+ self.load_batch_results
+ return self.nbr_of_docs
+
+ @cached_property
+ def available_results(self):
+ if not self.nbr_of_results:
+ self.load_batch_results
+ return self.nbr_of_results
+
+ @property
+ def q(self):
+ return self.query
+
+ def __eq__(self, other):
+ return self.page == other.page and self.query == other.query
+
+ def subset(self):
+ return self.batch
+
+ @property
+ def page_index(self):
+ return self.page
+
+ def page_by_index(self, index):
+ return SearchPostgres(self.request, self.query, index)
+
+ @cached_property
+ def batch(self):
+ if not self.query:
+ return None
+
+ if self.query.startswith('#'):
+ results = self.hashtag_search()
+ else:
+ results = self.generic_search()
+
+ return results[self.offset:self.offset + self.batch_size]
+
+ @cached_property
+ def load_batch_results(self):
+ """Load search results and sort events by latest occurrence.
+
+ This methods is a wrapper around `batch.load()`, which returns the
+ actual search results form the query. """
+
+ batch = self.batch
+ events = []
+ non_events = []
+ for search_result in batch:
+ if isinstance(search_result, Event):
+ events.append(search_result)
+ else:
+ non_events.append(search_result)
+ if not events:
+ return batch
+ sorted_events = sorted(events, key=lambda e: e.latest_occurrence.start)
+ return sorted_events + non_events
+
+ def generic_search(self):
+ doc_count = 0
+ results = []
+
+ language = locale_mapping(self.request.locale)
+ for model in searchable_sqlalchemy_models(Base):
+ if model.es_public or self.request.is_logged_in:
+ query = self.request.session.query(model)
+ doc_count += query.count()
+ query = query.filter(
+ model.fts_idx.op('@@')(func.websearch_to_tsquery(
+ language, self.query))
+ )
+ query = query.order_by(func.ts_rank_cd(
+ model.fts_idx, func.websearch_to_tsquery(language,
+ self.query)))
+ results.extend(query.all())
+
+ self.nbr_of_docs = doc_count
+ self.nbr_of_results = len(results)
+ results.sort(key=attrgetter('search_score'), reverse=False)
+ return results
+
+ def hashtag_search(self):
+ q = self.query.lstrip('#')
+ results = []
+
+ for model in searchable_sqlalchemy_models(Base):
+ # skip certain tables for hashtag search for better performance
+ if model.__tablename__ not in ['attendees', 'files', 'people',
+ 'tickets', 'users']:
+ if model.es_public or self.request.is_logged_in:
+ for doc in self.request.session.query(model).all():
+ if doc.es_tags and q in doc.es_tags:
+ results.append(doc)
+
+ self.nbr_of_results = len(results)
+ results.sort(key=attrgetter('search_score'), reverse=False)
+ return results
+
+ def feeling_lucky(self):
+ if self.batch:
+ first_entry = self.batch[0].load()
+
+ # XXX the default view to the event should be doing the redirect
+ if first_entry.__tablename__ == 'events':
+ return self.request.link(first_entry, 'latest')
+ else:
+ return self.request.link(first_entry)
+
+ @cached_property
+ def subset_count(self):
+ return self.available_results
+
+ def suggestions(self):
+ suggestions = list()
+
+ for element in self.generic_search():
+ suggest = getattr(element, 'es_suggestion', [])
+ suggestions.append(suggest)
+
+ return tuple(suggestions[:15])
diff --git a/src/onegov/org/path.py b/src/onegov/org/path.py
index 6daf6278fd..90349572a9 100644
--- a/src/onegov/org/path.py
+++ b/src/onegov/org/path.py
@@ -55,6 +55,7 @@
from onegov.org.models import ResourceRecipient
from onegov.org.models import ResourceRecipientCollection
from onegov.org.models import Search
+from onegov.org.models import SearchPostgres
from onegov.org.models import SiteCollection
from onegov.org.models import TicketNote
from onegov.org.models import Topic
@@ -506,6 +507,11 @@ def get_search(request, q='', page=0):
return Search(request, q, page)
+@OrgApp.path(model=SearchPostgres, path='/search_postgres')
+def get_postgres_search(request, q='', page=0):
+ return SearchPostgres(request, q, page)
+
+
@OrgApp.path(model=AtoZPages, path='/a-z')
def get_a_to_z(request):
return AtoZPages(request)
diff --git a/src/onegov/org/templates/search_postgres.pt b/src/onegov/org/templates/search_postgres.pt
new file mode 100644
index 0000000000..fd6a15c450
--- /dev/null
+++ b/src/onegov/org/templates/search_postgres.pt
@@ -0,0 +1,69 @@
+
+
+ ${title}
+
+
+
+
+ Postgres Searching is currently unavailable due to technical
+ difficulties.
+ Please excuse the inconvenience and try again later.
+
+
+
+
+
+
+ ${resultslabel}
+
+
+
Your postgres search returned no results.
+
+
+
+ -
+
+
+
+
+ - Score: ${result.explanation['score']}
+
+ -
+ ${title}: ${result.explanation[key]['value']}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/onegov/org/views/search.py b/src/onegov/org/views/search.py
index c172c66201..35e985fdb1 100644
--- a/src/onegov/org/views/search.py
+++ b/src/onegov/org/views/search.py
@@ -4,14 +4,13 @@
from onegov.org import _, OrgApp
from onegov.org.elements import Link
from onegov.org.layout import DefaultLayout
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
from onegov.search import SearchOfflineError
from webob import exc
@OrgApp.html(model=Search, template='search.pt', permission=Public)
def search(self, request, layout=None):
-
layout = layout or DefaultLayout(self, request)
layout.breadcrumbs.append(Link(_("Search"), '#'))
@@ -46,9 +45,56 @@ def search(self, request, layout=None):
}
+@OrgApp.html(model=SearchPostgres, template='search_postgres.pt',
+ permission=Public)
+def search_postgres(self, request, layout=None):
+ layout = layout or DefaultLayout(self, request)
+ layout.breadcrumbs.append(Link(_("Search"), '#'))
+
+ try:
+ searchlabel = _("Search through ${count} indexed documents", mapping={
+ 'count': self.available_documents
+ })
+ resultslabel = _("${count} Results", mapping={
+ 'count': self.available_results
+ })
+ except SearchOfflineError:
+ return {
+ 'title': _("Search Unavailable"),
+ 'layout': layout,
+ 'connection': False
+ }
+
+ if 'lucky' in request.GET:
+ url = self.feeling_lucky()
+
+ if url:
+ return morepath.redirect(url)
+
+ return {
+ # TODO switch back to 'Search' once es is gone
+ # 'title': _("Search"),
+ 'title': _("Org Search Postgres"),
+ 'model': self,
+ 'layout': layout,
+ 'hide_search_header': True,
+ 'searchlabel': searchlabel,
+ 'resultslabel': resultslabel,
+ 'connection': True
+ }
+
+
@OrgApp.json(model=Search, name='suggest', permission=Public)
def suggestions(self, request):
try:
return tuple(self.suggestions())
except SearchOfflineError as exception:
raise exc.HTTPNotFound() from exception
+
+
+@OrgApp.json(model=SearchPostgres, name='suggest', permission=Public)
+def suggestions_postgres(self, request):
+ try:
+ return tuple(self.suggestions())
+ except SearchOfflineError as exception:
+ raise exc.HTTPNotFound() from exception
diff --git a/src/onegov/page/model.py b/src/onegov/page/model.py
index 1a2fd7bec6..0d88ceb4f0 100644
--- a/src/onegov/page/model.py
+++ b/src/onegov/page/model.py
@@ -25,6 +25,10 @@ class Page(AdjacencyList, ContentMixin, TimestampMixin, UTCPublicationMixin):
__tablename__ = 'pages'
+ @property
+ def search_score(self) -> int:
+ return 2
+
if TYPE_CHECKING:
# we override these relationships to be more specific
parent: relationship['Page']
diff --git a/src/onegov/people/models/membership.py b/src/onegov/people/models/membership.py
index b1e3d41de7..46efeebe03 100644
--- a/src/onegov/people/models/membership.py
+++ b/src/onegov/people/models/membership.py
@@ -88,6 +88,10 @@ class AgencyMembership(Base, ContentMixin, TimestampMixin, ORMSearchable,
#: when the membership started
since = Column(Text, nullable=True)
+ @property
+ def search_score(self):
+ return 3
+
@property
def siblings_by_agency(self):
""" Returns a query that includes all siblings by agency, including
diff --git a/src/onegov/people/models/person.py b/src/onegov/people/models/person.py
index 0da149828e..6acc883628 100644
--- a/src/onegov/people/models/person.py
+++ b/src/onegov/people/models/person.py
@@ -1,11 +1,12 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.mixins import ContentMixin
from onegov.core.orm.mixins import TimestampMixin
from onegov.core.orm.mixins import UTCPublicationMixin
from onegov.core.orm.types import UUID
from onegov.search import ORMSearchable
-from sqlalchemy import Column
-from sqlalchemy import Text
+from sqlalchemy import Column, Text
from uuid import uuid4
from vobject import vCard
from vobject.vcard import Address
@@ -38,9 +39,9 @@ class Person(Base, ContentMixin, TimestampMixin, ORMSearchable,
@property
def es_suggestion(self):
- return (self.title, f"{self.first_name} {self.last_name}")
+ return self.title
- @property
+ @hybrid_property
def title(self):
""" Returns the Estern-ordered name. """
@@ -129,6 +130,10 @@ def spoken_title(self):
#: some remarks about the person
notes = Column(Text, nullable=True)
+ @property
+ def search_score(self):
+ return 3
+
def vcard_object(self, exclude=None, include_memberships=True):
""" Returns the person as vCard (3.0) object.
diff --git a/src/onegov/search/__init__.py b/src/onegov/search/__init__.py
index 24e6219b8d..f71bf05d93 100644
--- a/src/onegov/search/__init__.py
+++ b/src/onegov/search/__init__.py
@@ -4,11 +4,11 @@
from onegov.search.mixins import Searchable, ORMSearchable, SearchableContent
from onegov.search.dsl import Search
-from onegov.search.integration import ElasticsearchApp
+from onegov.search.integration import SearchApp
from onegov.search.errors import SearchOfflineError
__all__ = [
- 'ElasticsearchApp',
+ 'SearchApp',
'ORMSearchable',
'Search',
'Searchable',
diff --git a/src/onegov/search/cli.py b/src/onegov/search/cli.py
index c87cc86147..9b459f2620 100644
--- a/src/onegov/search/cli.py
+++ b/src/onegov/search/cli.py
@@ -5,7 +5,6 @@
from onegov.core.cli import command_group, pass_group_context
from sedate import utcnow
-
cli = command_group()
@@ -13,18 +12,26 @@
@click.option('--fail', is_flag=True, default=False, help='Fail on errors')
@pass_group_context
def reindex(group_context, fail):
- """ Reindexes all objects in the elasticsearch database. """
+ """ Reindexes all objects in the postgresql database. """
def run_reindex(request, app):
- if not hasattr(request.app, 'es_client'):
- return
+ """
+ Looping over all models in project deleting all full text search (
+ fts) indexes in postgresql and re-creating them
+ :param request: request
+ :param app: application context
+ """
title = f"Reindexing {request.app.application_id}"
print(click.style(title, underline=True))
start = utcnow()
- request.app.es_perform_reindex(fail)
+ app.psql_perform_reindex(request)
+ print(f"- psql indexing took {utcnow() - start}")
- print(f"took {utcnow() - start}")
+ # TODO: remove es indexing once es is gone
+ start = utcnow()
+ request.app.es_perform_reindex(fail)
+ print(f"- es indexing took {utcnow() - start}")
return run_reindex
diff --git a/src/onegov/search/integration.py b/src/onegov/search/integration.py
index 3880302fa2..adc973b133 100644
--- a/src/onegov/search/integration.py
+++ b/src/onegov/search/integration.py
@@ -10,6 +10,8 @@
from elasticsearch import TransportError
from elasticsearch.connection import create_ssl_context
from more.transaction.main import transaction_tween_factory
+
+from onegov.core.orm import Base
from onegov.search import Search, log
from onegov.search.errors import SearchOfflineError
from onegov.search.indexer import Indexer
@@ -90,8 +92,9 @@ def is_5xx_error(error):
return error.status_code and str(error.status_code).startswith('5')
-class ElasticsearchApp(morepath.App):
- """ Provides elasticsearch integration for
+# TODO: remove all es specific things ones es is gone
+class SearchApp(morepath.App):
+ """ Provides elasticsearch and postgres integration for
:class:`onegov.core.framework.Framework` based applications.
The application must be connected to a database.
@@ -326,6 +329,7 @@ def es_suggestions_by_request(self, request, query, types='*',
else:
languages = '*'
+ print(f'es_suggestion_by_request language: {languages}')
return self.es_suggestions(
query,
languages=languages,
@@ -394,8 +398,20 @@ def reindex_model(model):
self.es_indexer.bulk_process()
+ def psql_perform_reindex(self, request):
+ """ Re-indexes all `searchable' models in postgresql db ensuring
+ each table will be indexed only once.
+
+ """
+ done = []
+
+ for model in searchable_sqlalchemy_models(Base):
+ if model.__tablename__ not in done:
+ model.reindex(request, model)
+ done.append(model.__tablename__)
+
-@ElasticsearchApp.tween_factory(over=transaction_tween_factory)
+@SearchApp.tween_factory(over=transaction_tween_factory)
def process_indexer_tween_factory(app, handler):
def process_indexer_tween(request):
diff --git a/src/onegov/search/mixins.py b/src/onegov/search/mixins.py
index e6cfb01149..b1f2b644fc 100644
--- a/src/onegov/search/mixins.py
+++ b/src/onegov/search/mixins.py
@@ -1,6 +1,15 @@
-from onegov.search.utils import classproperty
+from sqlalchemy import Column, func, Computed # type:ignore[attr-defined]
+from sqlalchemy.dialects.postgresql import TSVECTOR
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import deferred
+
+from onegov.core.upgrade import UpgradeContext
+from onegov.search.utils import classproperty, \
+ get_fts_index_localized_languages, get_fts_index_basic_languages
from onegov.search.utils import extract_hashtags
+from typing import Any, TYPE_CHECKING
+
class Searchable:
""" Defines the interface required for an object to be searchable.
@@ -38,6 +47,18 @@ def es_type_name(self):
identity is a completely different model.
"""
+ TEXT_SEARCH_COLUMN_NAME = 'fts_idx'
+
+ if TYPE_CHECKING:
+ fts_idx: 'Column[dict[str, Any]]'
+
+ # column for full text search index
+ @declared_attr # type:ignore[no-redef]
+ def fts_idx(cls) -> 'Column[dict[str, Any]]':
+ col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+ if hasattr(cls, '__table__') and hasattr(cls.__table__.c, col_name):
+ return deferred(cls.__table__.c.fts_idx)
+ return deferred(Column(col_name, TSVECTOR))
@classproperty
def es_properties(self):
@@ -134,7 +155,106 @@ def es_tags(self):
""" Returns a list of tags associated with this content. """
return None
+ @property
+ def search_score(self):
+ """
+ the lower the score the higher the class type will be shown in search
+ results. Default is 10 (lowest)
+ """
+ return 10
+
+ @staticmethod
+ def psql_tsvector_expression(model):
+ """
+ Provides the tsvector expression for postgres for the defined
+ model. Depending on the model columns and properties are used for full
+ text search index.
+
+ :return: tsvector expression
+ """
+ objects = [getattr(model, p) for p in model.es_properties if
+ not p.startswith('es_')]
+ return Searchable.create_tsvector_expression(*objects)
+
+ @staticmethod
+ def reindex(request, model):
+ """
+ Re-indexes the table by dropping and adding the full text search
+ column.
+ """
+ Searchable.drop_fts_column(request, model)
+ Searchable.add_fts_column(request, model)
+
+ @staticmethod
+ def drop_fts_column(request, model):
+ """
+ Drops the full text search column
+
+ :param request: request object
+ :param model: model to drop the index from
+ :return: None
+ """
+
+ col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+ context = UpgradeContext(request)
+
+ if context.has_column(model.__tablename__, col_name):
+ context.operations.drop_column(model.__tablename__, col_name)
+
+ @staticmethod
+ def add_fts_column(request, model):
+
+ """
+ This function is used for re-indexing and as migration step moving to
+ postgresql full text search (fts), OGC-508.
+
+ It adds a separate column for the tsvector to `schema`.`table`
+ creating a multilingual gin index on the columns/data defined per
+ model.
+
+ :param request: request object
+ :param model: model to add the index
+ :return: None
+ """
+ col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+ context = UpgradeContext(request)
+ if not context.has_column(model.__tablename__, col_name):
+ tsvector_expression = None
+ for prop_name, type_info in model.es_properties.items():
+ if not prop_name.startswith('es_'):
+ prop_type = type_info.get('type', None)
+ prop = getattr(model, prop_name)
+ languages = get_fts_index_basic_languages()
+
+ if prop_type in ['localized', 'localized_html']:
+ # only for 'localized' properties we create the
+ # index localized
+ languages.extend(get_fts_index_localized_languages())
+
+ for language in languages:
+ expr = func.to_tsvector(language,
+ func.coalesce(prop, ''))
+
+ if tsvector_expression is None:
+ tsvector_expression = expr
+ else:
+ tsvector_expression = tsvector_expression.concat(
+ expr)
+
+ context.operations.add_column(
+ model.__tablename__,
+ Column(col_name,
+ TSVECTOR,
+ Computed(
+ tsvector_expression,
+ persisted=True),
+ )
+ )
+ context.operations.execute("COMMIT")
+
+
+# TODO: rename prefix 'es' to 'ts' for text search
class ORMSearchable(Searchable):
""" Extends the default :class:`Searchable` class with sensible defaults
for SQLAlchemy orm models.
@@ -154,6 +274,7 @@ def es_last_change(self):
return getattr(self, 'last_change', None)
+# TODO: rename prefix 'es' to 'ts' for text search
class SearchableContent(ORMSearchable):
""" Adds search to all classes using the core's content mixin:
:class:`onegov.core.orm.mixins.content.ContentMixin`
diff --git a/src/onegov/search/utils.py b/src/onegov/search/utils.py
index 8d02ebe00e..1457251f58 100644
--- a/src/onegov/search/utils.py
+++ b/src/onegov/search/utils.py
@@ -8,7 +8,6 @@
from langdetect.utils.lang_profile import LangProfile
from onegov.core.orm import find_models
-
# XXX this is doubly defined in onegov.org.utils, maybe move to a common
# regex module in in onegov.core
HASHTAG = re.compile(r'#\w{3,}')
@@ -30,6 +29,16 @@ def searchable_sqlalchemy_models(base):
_invalid_index_characters = re.compile(r'[\\/?"<>|\s,A-Z:]+')
+def get_fts_index_languages():
+ """ Define index creation languages for full text search as we have a
+ limited set of used languages.
+
+ NOTE: 'simple' is used for tag, label or phrase searches
+
+ """
+ return ['simple', 'german', 'french', 'italian', 'english']
+
+
def is_valid_index_name(name):
""" Checks if the given name is a valid elasticsearch index name.
Elasticsearch does it's own checks, but we can do it earlier and we are
diff --git a/src/onegov/ticket/model.py b/src/onegov/ticket/model.py
index 538f3e9b79..885045130e 100644
--- a/src/onegov/ticket/model.py
+++ b/src/onegov/ticket/model.py
@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
from onegov.core.orm import Base
from onegov.core.orm.mixins import TimestampMixin
from onegov.core.orm.types import JSON, UUID
@@ -105,6 +107,10 @@ class Ticket(Base, TimestampMixin, ORMSearchable):
#: true if the notifications for this ticket should be muted
muted: 'Column[bool]' = Column(Boolean, nullable=False, default=False)
+ @property
+ def search_score(self) -> int:
+ return 6
+
if TYPE_CHECKING:
created: Column[datetime]
else:
@@ -142,6 +148,7 @@ def created(cls) -> 'Column[datetime]':
# limit the search to the ticket number -> the rest can be found
es_public = False
+
es_properties = {
'number': {'type': 'text'},
'title': {'type': 'text'},
@@ -152,7 +159,7 @@ def created(cls) -> 'Column[datetime]':
'extra_localized_text': {'type': 'localized'}
}
- @property
+ @hybrid_property
def extra_localized_text(self) -> str | None:
""" Maybe used by child-classes to return localized extra data that
should be indexed as well.
@@ -167,14 +174,14 @@ def es_suggestion(self) -> list[str]:
self.number.replace('-', '')
]
- @property
+ @hybrid_property
def ticket_email(self) -> str | None:
if self.handler.deleted:
return self.snapshot.get('email')
else:
return self.handler.email
- @property
+ @hybrid_property
def ticket_data(self) -> 'Sequence[str] | None':
if self.handler.deleted:
return self.snapshot.get('summary')
diff --git a/src/onegov/town6/templates/search_postgres.pt b/src/onegov/town6/templates/search_postgres.pt
new file mode 100644
index 0000000000..055758f94a
--- /dev/null
+++ b/src/onegov/town6/templates/search_postgres.pt
@@ -0,0 +1,70 @@
+
+
+ ${title}
+
+
+
+
+ Postgres Searching is currently unavailable due to technical
+ difficulties.
+ Please excuse the inconvenience and try again later.
+
+
+
+
+
+
+ ${resultslabel}
+
+
+
+
Your postgres search returned no results.
+
+
+
+ -
+
+
+
+
+ - Score: ${result.explanation['score']}
+
+ -
+ ${title}: ${result.explanation[key]['value']}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/onegov/town6/views/search.py b/src/onegov/town6/views/search.py
index 6e7b1d71d4..14d5bdc62d 100644
--- a/src/onegov/town6/views/search.py
+++ b/src/onegov/town6/views/search.py
@@ -1,10 +1,16 @@
from onegov.core.security import Public
-from onegov.org.views.search import search
+from onegov.org.views.search import search, search_postgres
from onegov.town6 import TownApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
from onegov.town6.layout import DefaultLayout
@TownApp.html(model=Search, template='search.pt', permission=Public)
def town_search(self, request):
return search(self, request, DefaultLayout(self, request))
+
+
+@TownApp.html(model=SearchPostgres, template='search_postgres.pt',
+ permission=Public)
+def town_search_postgres(self, request):
+ return search_postgres(self, request, DefaultLayout(self, request))
diff --git a/src/onegov/translator_directory/models/translator.py b/src/onegov/translator_directory/models/translator.py
index 82b5e60a14..53f4dcfd60 100644
--- a/src/onegov/translator_directory/models/translator.py
+++ b/src/onegov/translator_directory/models/translator.py
@@ -28,6 +28,7 @@
from .language import Language
+# TODO rename to ts (text search)
class ESMixin(ORMSearchable):
es_properties = {
@@ -182,6 +183,10 @@ class Translator(Base, TimestampMixin, AssociatedFiles, ContentMixin,
expertise_professional_guilds_other: 'dict_property[Sequence[str]]'
expertise_professional_guilds_other = meta_property(default=tuple)
+ @property
+ def search_score(self):
+ return 4
+
@property
def expertise_professional_guilds_all(self):
return (
diff --git a/src/onegov/user/models/user.py b/src/onegov/user/models/user.py
index c1eee656c1..70eb73f36d 100644
--- a/src/onegov/user/models/user.py
+++ b/src/onegov/user/models/user.py
@@ -1,4 +1,5 @@
from datetime import datetime
+
from onegov.core.crypto import hash_password, verify_password
from onegov.core.orm import Base
from onegov.core.orm.mixins import data_property, dict_property, TimestampMixin
@@ -9,8 +10,7 @@
from onegov.core.utils import yubikey_otp_to_serial
from onegov.search import ORMSearchable
from onegov.user.models.group import UserGroup
-from sqlalchemy import Boolean, Column, Index, Text, func, ForeignKey
-from sqlalchemy import UniqueConstraint
+from sqlalchemy import Boolean, Column, Text, func, ForeignKey
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import backref, deferred, relationship
from uuid import uuid4, UUID as UUIDType
@@ -57,7 +57,7 @@ class User(Base, TimestampMixin, ORMSearchable):
def es_suggestion(self) -> tuple[str, str]:
return (self.realname or self.username, self.username)
- @property
+ @hybrid_property
def userprofile(self) -> list[str]:
if not self.data:
return []
@@ -150,10 +150,9 @@ def userprofile(self) -> list[str]:
signup_token: 'Column[str | None]' = Column(
Text, nullable=True, default=None)
- __table_args__ = (
- Index('lowercase_username', func.lower(username), unique=True),
- UniqueConstraint('source', 'source_id', name='unique_source_id'),
- )
+ @property
+ def search_score(self) -> int:
+ return 5
if TYPE_CHECKING:
# HACK: This probably won't be necessary in SQLAlchemy 2.0, but
diff --git a/tests/onegov/search/test_indexer.py b/tests/onegov/search/test_indexer.py
index 8a1fe21104..3ad31b52bb 100644
--- a/tests/onegov/search/test_indexer.py
+++ b/tests/onegov/search/test_indexer.py
@@ -2,6 +2,11 @@
import pytest
from datetime import datetime
+
+from onegov.core.orm import Base
+from onegov.directory import DirectoryEntry
+from onegov.org.models import Topic
+from onegov.people import Agency
from onegov.search import Searchable, SearchOfflineError, utils
from onegov.search.indexer import parse_index_name
from onegov.search.indexer import (
@@ -14,6 +19,10 @@
from queue import Queue
from unittest.mock import Mock
+from onegov.search.utils import searchable_sqlalchemy_models
+from onegov.ticket import Ticket
+from onegov.user import User
+
def test_index_manager_assertions(es_client):
@@ -717,3 +726,88 @@ def test_elasticsearch_outage(es_client, es_url):
indexer.es_client.indices.refresh(index='_all')
assert indexer.es_client\
.search(index='_all')['hits']['total']['value'] == 2
+
+
+def test_psql_tsvector_string():
+ assert Searchable.create_tsvector_string(('col_lower')) == \
+ "'func.coalesce(col_lower, '')'"
+
+ # FIXME: implement lower
+ # assert Searchable.create_tsvector_string(['Col_Higher']) == \
+ # 'coalesce("\'col_higher\'", \'\')'
+
+ assert Searchable.create_tsvector_string('col_a', 'col_b') == \
+ "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'"
+
+ assert Searchable.create_tsvector_string('a', 'b', 'c') == \
+ "'func.coalesce(a, '')' || ' ' || 'func.coalesce(b, '')' || ' ' || " \
+ "'func.coalesce(c, '')'"
+
+ cols = ['col_a', 'col_b']
+ assert Searchable.create_tsvector_string(*cols) == \
+ "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'"
+
+
+def test_multi_language_tsvector_expression(monkeypatch):
+ tsvector_string = "'func.coalesce(my_col, '')'"
+ x = Searchable.multi_language_tsvector_expression(tsvector_string)
+ assert x == "to_tsvector('simple', 'func.coalesce(my_col, '')') || " \
+ "to_tsvector('german', 'func.coalesce(my_col, '')') || " \
+ "to_tsvector('french', 'func.coalesce(my_col, '')') || " \
+ "to_tsvector('italian', 'func.coalesce(my_col, '')') || " \
+ "to_tsvector('english', 'func.coalesce(my_col, '')')"
+
+ def fake():
+ return ['simple']
+ tsvector_string = "'func.coalesce(group, '')'"
+ monkeypatch.setattr(utils, 'get_fts_index_languages', fake)
+ assert Searchable.multi_language_tsvector_expression(
+ tsvector_string) == "to_tsvector('simple', 'func.coalesce(group, '')')"
+
+
+def test_psql_tsvector_string_generation_models():
+ count = 0
+
+ for model in searchable_sqlalchemy_models(Base):
+ print(f'model {model}..')
+ tsvector = model.psql_tsvector_string(model)
+ for p in getattr(model, 'es_properties', []):
+ if p in model.__dict__ and not p.startswith('_es'):
+ # verify all properties are reflected in the tsvector
+ assert p in tsvector
+
+ # random sample
+ if model == Agency:
+ count += 1
+ assert tsvector == "'func.coalesce(title, '')' || ' ' || " \
+ "'func.coalesce(description, '')' || ' ' || " \
+ "'func.coalesce(portrait, '')'"
+ elif model == User:
+ count += 1
+ assert tsvector == "'func.coalesce(username, '')' || ' ' || " \
+ "'func.coalesce(realname, '')' || ' ' || " \
+ "'func.coalesce(userprofile, '')'"
+ elif model == DirectoryEntry:
+ count += 1
+ assert tsvector == "'func.coalesce(keywords, '')' || ' ' || " \
+ "'func.coalesce(title, '')' || ' ' || " \
+ "'func.coalesce(lead, '')' || ' ' || " \
+ "'func.coalesce(directory_id, '')' || ' ' || " \
+ "'func.coalesce(text, '')'"
+ elif model == Ticket:
+ count += 1
+ assert tsvector == "'func.coalesce(number, '')' || ' ' || " \
+ "'func.coalesce(title, '')' || ' ' || " \
+ "'func.coalesce(subtitle, '')' || ' ' || " \
+ "'func.coalesce(group, '')' || ' ' || " \
+ "'func.coalesce(ticket_email, '')' || ' ' || " \
+ "'func.coalesce(ticket_data, '')' || ' ' || " \
+ "'func.coalesce(extra_localized_text, '')'"
+ elif model == Topic:
+ count += 1
+ assert tsvector == "'func.coalesce(title, '')' || ' ' || " \
+ "'func.coalesce(lead, '')' || ' ' || " \
+ "'func.coalesce(text, '')'"
+
+ # verify if statements reached and tested
+ assert count == 5
diff --git a/tests/onegov/search/test_integration.py b/tests/onegov/search/test_integration.py
index 3280604079..a018434c6c 100644
--- a/tests/onegov/search/test_integration.py
+++ b/tests/onegov/search/test_integration.py
@@ -5,10 +5,11 @@
from datetime import timedelta
from elasticsearch_dsl.function import SF
from elasticsearch_dsl.query import MatchPhrase, FunctionScore
+
from onegov.core import Framework
from onegov.core.orm.mixins import TimestampMixin
from onegov.core.utils import scan_morepath_modules
-from onegov.search import ElasticsearchApp, ORMSearchable
+from onegov.search import SearchApp, ORMSearchable
from sqlalchemy import Boolean, Column, Integer, Text
from sqlalchemy.ext.declarative import declarative_base
from webtest import TestApp as Client
@@ -17,7 +18,7 @@
def test_app_integration(es_url):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
app = App()
@@ -33,7 +34,7 @@ class App(Framework, ElasticsearchApp):
def test_search_query(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -46,12 +47,20 @@ class Document(Base, ORMSearchable):
body = Column(Text, nullable=True)
public = Column(Boolean, nullable=False)
language = Column(Text, nullable=False)
+ # fts_idx = Column(TSVECTOR, Computed('', persisted=True))
+ # __table_args__ = (
+ # Index('fts_idx', fts_idx, postgresql_using='gin'),
+ # )
es_properties = {
'title': {'type': 'localized'},
'body': {'type': 'localized'}
}
+ # @staticmethod
+ # def psql_tsvector_string():
+ # return Searchable.create_tsvector_string('title', 'body')
+
@property
def es_suggestion(self):
return self.title
@@ -148,10 +157,17 @@ def es_language(self):
assert document.title == "Öffentlich"
assert document.public
+ ##################
+ # postgresql tests
+ # app.psql_perform_reindex()
+
+ # results = app.psql_search('')
+ # assert results
+
def test_orm_integration(es_url, postgres_dsn, redis_url):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -294,7 +310,7 @@ def view_delete_document(self, request):
def test_alternate_id_property(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -356,7 +372,7 @@ def es_suggestion(self):
def test_orm_polymorphic(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -442,7 +458,7 @@ def update():
def test_orm_polymorphic_sublcass_only(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -498,7 +514,7 @@ def es_suggestion(self):
def test_suggestions(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -617,7 +633,7 @@ def es_suggestion(self):
def test_language_detection(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -671,7 +687,7 @@ class Document(Base, ORMSearchable):
def test_language_update(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
@@ -725,7 +741,7 @@ class Document(Base, ORMSearchable):
def test_date_decay(es_url, postgres_dsn):
- class App(Framework, ElasticsearchApp):
+ class App(Framework, SearchApp):
pass
Base = declarative_base()
diff --git a/tests/onegov/search/test_utils.py b/tests/onegov/search/test_utils.py
index f5c29c3a97..fb88ea10e4 100644
--- a/tests/onegov/search/test_utils.py
+++ b/tests/onegov/search/test_utils.py
@@ -1,5 +1,6 @@
-from onegov.search import ORMSearchable, Searchable
+from onegov.search import ORMSearchable
from onegov.search import utils
+from onegov.search.mixins import Searchable
from sqlalchemy import Column, Integer, Text
from sqlalchemy.ext.declarative import declarative_base
@@ -101,3 +102,13 @@ class News(Page):
es_type_name = 'news'
assert utils.related_types(Page) == {'news', 'topic'}
+
+
+def test_create_tsvector_string():
+ assert Searchable.create_tsvector_string('username') == \
+ "coalesce(username, '')"
+ assert Searchable.create_tsvector_string('title', 'body') == \
+ "coalesce(title, '') || ' ' || coalesce(body, '')"
+ assert Searchable.create_tsvector_string('alpha', 'beta', 'gamma') == \
+ "coalesce(alpha, '') || ' ' || coalesce(beta, '') || ' ' || " \
+ "coalesce(gamma, '')"
diff --git a/tests/shared/fixtures.py b/tests/shared/fixtures.py
index 5d911910e5..5ecf246805 100644
--- a/tests/shared/fixtures.py
+++ b/tests/shared/fixtures.py
@@ -36,7 +36,7 @@
from threading import Thread
from uuid import uuid4
from webdriver_manager.chrome import ChromeDriverManager
-from webdriver_manager.core.os_manager import ChromeType
+# from webdriver_manager.core.os_manager import ChromeType
redis_path = which('redis-server')