Features/285 add redis (#287)

* Add redis and associated env variables
* Cache events for up to 15 minutes:
* Add BaseConnector with some helper functions to get/put cached events into redis.
* Move some password hashing/string encryption funcs to a utils.py file.
* Add redis_instance as a constructor variable.
* Add subscriber_id as a constructor variable for CalDavConnector.
* Add model_(load/dump)_redis helper function to Event schema.
* Add test for model_(load/dump)_redis helper functions.
* Add redis key define.
* Add redis dependency injection function.
* Require calendar_id (pk) and subscriber_id (pk) for each calendar connector
* Cache key to allow for busting all or a specific calendar from a specific user
* Bust event cache when a calendar is synced or an event is created or removed.
* Lots of wiring up new parameters / mock parameters for tests
* Add caching to the booking page:
* Use the same event cache for the booking page.
* Verify that a user can book a given time slot against remote calendar events.
* Add tests to request availability.
* Rename REDIS_REMOVE_EVENTS_KEY to REDIS_REMOTE_EVENTS_KEY
* Rename get_cache_events to get_cached_events
* Cache result of setup_encryption_engine

backend/.env.example

@@ -85,3 +85,13 @@ CALDAV_TEST_USER=
 CALDAV_TEST_PASS=
 GOOGLE_TEST_USER=
 GOOGLE_TEST_PASS=
+
+# -- Redis --
+REDIS_URL=redis
+REDIS_PORT=6379
+REDIS_DB=0
+# No value = Python None
+REDIS_PASSWORD
+
+# In minutes, the time a cached remote event will expire at.
+REDIS_EVENT_EXPIRE_TIME=15

backend/.env.test

@@ -88,3 +88,13 @@ GOOGLE_TEST_USER=
 GOOGLE_TEST_PASS=
 
 TEST_USER_EMAIL=test@example.org
+
+# -- Redis --
+REDIS_URL=localhost
+REDIS_PORT=6379
+REDIS_DB=0
+# No value = Python None
+REDIS_PASSWORD
+
+# In minutes, the time a cached remote event will expire at.
+REDIS_EVENT_EXPIRE_TIME=15

backend/requirements.txt

@@ -28,3 +28,5 @@ uvicorn==0.20.0
 validators==0.20.0
 oauthlib==3.2.2
 requests-oauthlib==1.3.1
+redis==5.0.2
+hiredis==2.3.2

backend/src/appointment/controller/calendar.py

@@ -3,46 +3,113 @@
 Handle connection to a CalDAV server.
 """
 import json
+import os
 
 import caldav.lib.error
 import requests
+from redis import Redis
 from caldav import DAVClient
 from fastapi import BackgroundTasks
 from google.oauth2.credentials import Credentials
 from icalendar import Calendar, Event, vCalAddress, vText
 from datetime import datetime, timedelta, timezone, UTC
-from zoneinfo import ZoneInfo
-from dateutil.parser import parse
 
+from .. import utils
+from ..defines import REDIS_REMOTE_EVENTS_KEY, DATEFMT
 from .apis.google_client import GoogleClient
 from ..database import schemas, models
 from ..database.models import CalendarProvider
-from ..controller.mailer import Attachment, InvitationMail
+from ..controller.mailer import Attachment
 from ..l10n import l10n
 from ..tasks.emails import send_invite_email
 
-DATEFMT = "%Y-%m-%d"
 
+class BaseConnector:
+    redis_instance: Redis | None
+    subscriber_id: int
+    calendar_id: int
 
-class GoogleConnector:
+    def __init__(self, subscriber_id: int, calendar_id: int | None, redis_instance: Redis | None = None):
+        self.redis_instance = redis_instance
+        self.subscriber_id = subscriber_id
+        self.calendar_id = calendar_id
+
+    def obscure_key(self, key):
+        """Obscure part of a key with our encryption algo"""
+        return utils.setup_encryption_engine().encrypt(key)
+
+    def get_key_body(self, only_subscriber = False):
+        parts = [self.obscure_key(self.subscriber_id)]
+        if not only_subscriber:
+            parts.append(self.obscure_key(self.calendar_id))
+
+        return ":".join(parts)
+
+    def get_cached_events(self, key_scope):
+        """Retrieve any cached events, else returns None if redis is not available or there's no cache."""
+        if self.redis_instance is None:
+            return None
+
+        key_scope = self.obscure_key(key_scope)
+
+        encrypted_events = self.redis_instance.get(f'{REDIS_REMOTE_EVENTS_KEY}:{self.get_key_body()}:{key_scope}')
+        if encrypted_events is None:
+            return None
+
+        return [schemas.Event.model_load_redis(blob) for blob in json.loads(encrypted_events)]
+
+    def put_cached_events(self, key_scope, events: list[schemas.Event], expiry=os.getenv('REDIS_EVENT_EXPIRE_SECONDS')):
+        """Sets the passed cached events with an option to set a custom expiry time."""
+        if self.redis_instance is None:
+            return False
+
+        key_scope = self.obscure_key(key_scope)
+
+        encrypted_events = json.dumps([event.model_dump_redis() for event in events])
+        self.redis_instance.set(f'{REDIS_REMOTE_EVENTS_KEY}:{self.get_key_body()}:{key_scope}',
+                                value=encrypted_events, ex=expiry)
+
+        return True
+
+    def bust_cached_events(self, all_calendars = False):
+        """Delete cached events for a specific subscriber/calendar. 
+        Optionally pass in all_calendars to remove all cached calendar events for a specific subscriber."""
+        if self.redis_instance is None:
+            return False
+
+        # Scan returns a tuple like: (Cursor start, [...keys found])
+        ret = self.redis_instance.scan(0, f'{REDIS_REMOTE_EVENTS_KEY}:{self.get_key_body(only_subscriber=all_calendars)}:*')
+
+        if len(ret[1]) == 0:
+            return False
+
+        # Expand the list in position 1, which is a list of keys found from the scan
+        self.redis_instance.delete(*ret[1])
+
+        return True
+
+
+class GoogleConnector(BaseConnector):
     """Generic interface for Google Calendar REST API.
        This should match CaldavConnector (except for the constructor).
     """
 
     def __init__(
         self,
+        subscriber_id,
+        calendar_id,
+        redis_instance,
         db,
+        remote_calendar_id,
         google_client: GoogleClient,
-        calendar_id,
-        subscriber_id,
         google_tkn: str = None,
     ):
-        # store credentials of remote location
+        super().__init__(subscriber_id, calendar_id, redis_instance)
+
         self.db = db
         self.google_client = google_client
         self.provider = CalendarProvider.google
-        self.calendar_id = calendar_id
-        self.subscriber_id = subscriber_id
+        self.remote_calendar_id = remote_calendar_id
         self.google_token = None
         # Create the creds class from our token (requires a refresh token)
         if google_tkn:
@@ -57,6 +124,8 @@ def sync_calendars(self):
 
         # We only support google right now!
         self.google_client.sync_calendars(db=self.db, subscriber_id=self.subscriber_id, token=self.google_token)
+        # We should refresh any events we might have for every calendar
+        self.bust_cached_events(all_calendars=True)
 
     def list_calendars(self):
         """find all calendars on the remote server"""
@@ -75,11 +144,16 @@ def list_calendars(self):
 
     def list_events(self, start, end):
         """find all events in given date range on the remote server"""
+        cache_scope = f"{start}_{end}"
+        cached_events = self.get_cached_events(cache_scope)
+        if cached_events:
+            return cached_events
+
         time_min = datetime.strptime(start, DATEFMT).isoformat() + "Z"
         time_max = datetime.strptime(end, DATEFMT).isoformat() + "Z"
 
         # We're storing google cal id in user...for now.
-        remote_events = self.google_client.list_events(self.calendar_id, time_min, time_max, self.google_token)
+        remote_events = self.google_client.list_events(self.remote_calendar_id, time_min, time_max, self.google_token)
 
         events = []
         for event in remote_events:
@@ -100,8 +174,10 @@ def list_events(self, start, end):
 
             all_day = "date" in event.get("start")
 
-            start = datetime.strptime(event.get("start")["date"], DATEFMT) if all_day else datetime.fromisoformat(event.get("start")["dateTime"])
-            end = datetime.strptime(event.get("end")["date"], DATEFMT) if all_day else datetime.fromisoformat(event.get("end")["dateTime"])
+            start = datetime.strptime(event.get("start")["date"], DATEFMT) if all_day else datetime.fromisoformat(
+                event.get("start")["dateTime"])
+            end = datetime.strptime(event.get("end")["date"], DATEFMT) if all_day else datetime.fromisoformat(
+                event.get("end")["dateTime"])
 
             events.append(
                 schemas.Event(
@@ -114,6 +190,8 @@ def list_events(self, start, end):
                 )
             )
 
+        self.put_cached_events(cache_scope, events)
+
         return events
 
     def create_event(
@@ -145,6 +223,9 @@ def create_event(
             ],
         }
         self.google_client.create_event(calendar_id=self.calendar_id, body=body, token=self.google_token)
+
+        self.bust_cached_events()
+
         return event
 
     def delete_events(self, start):
@@ -154,13 +235,14 @@ def delete_events(self, start):
         pass
 
 
-class CalDavConnector:
-    def __init__(self, url: str, user: str, password: str):
-        # store credentials of remote location
+class CalDavConnector(BaseConnector):
+    def __init__(self, subscriber_id: int, calendar_id: int, redis_instance, url: str, user: str, password: str):
+        super().__init__(subscriber_id, calendar_id, redis_instance)
+
         self.provider = CalendarProvider.caldav
         self.url = url
-        self.user = user
         self.password = password
+        self.user = user
         # connect to CalDAV server
         self.client = DAVClient(url=url, username=user, password=password)
 
@@ -181,7 +263,8 @@ def test_connection(self) -> bool:
         return 'VEVENT' in supported_comps
 
     def sync_calendars(self):
-        pass
+        # We don't sync anything for caldav, but might as well bust event cache.
+        self.bust_cached_events(all_calendars=True)
 
     def list_calendars(self):
         """find all calendars on the remote server"""
@@ -199,6 +282,11 @@ def list_calendars(self):
 
     def list_events(self, start, end):
         """find all events in given date range on the remote server"""
+        cache_scope = f"{start}_{end}"
+        cached_events = self.get_cached_events(cache_scope)
+        if cached_events:
+            return cached_events
+
         events = []
         calendar = self.client.calendar(url=self.url)
         result = calendar.search(
@@ -227,6 +315,9 @@ def list_events(self, start, end):
                     description=e.icalendar_component["description"] if "description" in e.icalendar_component else "",
                 )
             )
+
+        self.put_cached_events(cache_scope, events)
+
         return events
 
     def create_event(
@@ -249,6 +340,9 @@ def create_event(
         caldavEvent.add_attendee((organizer.name, organizer.email))
         caldavEvent.add_attendee((attendee.name, attendee.email))
         caldavEvent.save()
+
+        self.bust_cached_events()
+
         return event
 
     def delete_events(self, start):
@@ -262,6 +356,9 @@ def delete_events(self, start):
             if str(e.vobject_instance.vevent.dtstart.value).startswith(start):
                 e.delete()
                 count += 1
+
+        self.bust_cached_events()
+
         return count
 
 
@@ -329,7 +426,8 @@ def available_slots_from_schedule(s: models.Schedule) -> list[schemas.SlotBase]:
         farthest_booking = now + timedelta(days=1, minutes=s.farthest_booking)
 
         schedule_start = max([datetime.combine(s.start_date, s.start_time), earliest_booking])
-        schedule_end = min([datetime.combine(s.end_date, s.end_time), farthest_booking]) if s.end_date else farthest_booking
+        schedule_end = min(
+            [datetime.combine(s.end_date, s.end_time), farthest_booking]) if s.end_date else farthest_booking
 
         start_time = datetime.combine(now.min, s.start_time) - datetime.min
         end_time = datetime.combine(now.min, s.end_time) - datetime.min
@@ -353,7 +451,8 @@ def available_slots_from_schedule(s: models.Schedule) -> list[schemas.SlotBase]:
                 # We just loop through the start and end time and step by slot duration in seconds.
                 slots += [
                     schemas.SlotBase(start=current_datetime + timedelta(seconds=time), duration=s.slot_duration)
-                    for time in range(int(start_time.total_seconds()), int(end_time.total_seconds()), s.slot_duration * 60)
+                    for time in
+                    range(int(start_time.total_seconds()), int(end_time.total_seconds()), s.slot_duration * 60)
                 ]
 
         return slots
@@ -386,7 +485,8 @@ def existing_events_for_schedule(
         calendars: list[schemas.Calendar],
         subscriber: schemas.Subscriber,
         google_client: GoogleClient,
-        db
+        db,
+        redis = None
     ) -> list[schemas.Event]:
         """This helper retrieves all events existing in given calendars for the scheduled date range
         """
@@ -397,21 +497,31 @@ def existing_events_for_schedule(
             if calendar.provider == CalendarProvider.google:
                 con = GoogleConnector(
                     db=db,
+                    redis_instance=redis,
                     google_client=google_client,
-                    calendar_id=calendar.user,
+                    remote_calendar_id=calendar.user,
+                    calendar_id=calendar.id,
                     subscriber_id=subscriber.id,
                     google_tkn=subscriber.google_tkn,
                 )
             else:
-                con = CalDavConnector(calendar.url, calendar.user, calendar.password)
+                con = CalDavConnector(
+                    redis_instance=redis,
+                    url=calendar.url,
+                    user=calendar.user,
+                    password=calendar.password,
+                    subscriber_id=subscriber.id,
+                    calendar_id=calendar.id,
+                )
 
             now = datetime.now()
 
             earliest_booking = now + timedelta(minutes=schedule.earliest_booking)
             farthest_booking = now + timedelta(minutes=schedule.farthest_booking)
 
             start = max([datetime.combine(schedule.start_date, schedule.start_time), earliest_booking])
-            end = min([datetime.combine(schedule.end_date, schedule.end_time), farthest_booking]) if schedule.end_date else farthest_booking
+            end = min([datetime.combine(schedule.end_date, schedule.end_time),
+                       farthest_booking]) if schedule.end_date else farthest_booking
 
             try:
                 existing_events.extend(con.list_events(start.strftime(DATEFMT), end.strftime(DATEFMT)))

backend/src/appointment/database/schemas.py

@@ -2,10 +2,12 @@
 
 Definitions of valid data shapes for database and query models.
 """
+import json
 from datetime import datetime, date, time
 from typing import Annotated
 
 from pydantic import BaseModel, Field
+
 from .models import (
     AppointmentStatus,
     BookingStatus,
@@ -17,7 +19,7 @@
     ExternalConnectionType,
     MeetingLinkProviderType,
 )
-
+from .. import utils
 
 """ ATTENDEE model schemas
 """
@@ -284,6 +286,23 @@ class Event(BaseModel):
     calendar_color: str | None = None
     location: EventLocation | None = None
 
+    """Ideally this would just be a mixin, but I'm having issues figuring out a good
+    static constructor that will work for anything."""
+    def model_dump_redis(self):
+        """Dumps our event into an encrypted json blob for redis"""
+        values_json = self.model_dump_json()
+
+        return utils.setup_encryption_engine().encrypt(values_json)
+
+    @staticmethod
+    def model_load_redis(encrypted_blob):
+        """Loads and decrypts our encrypted json blob from redis"""
+
+        values_json = utils.setup_encryption_engine().decrypt(encrypted_blob)
+        values = json.loads(values_json)
+
+        return Event(**values)
+
 
 class FileDownload(BaseModel):
     name: str