Merge pull request #250 from norcalipa/norcalipa/view-caching-2

Adding additional caching, moving cache TTL to ENV

.env-prod

@@ -6,3 +6,4 @@ DB_USER=crank
 DB_PORT=3306
 PYTHON_UNBUFFERED=1
 REDIS_URL=redis://redis:6379/0
+CACHE_TTL=60

.github/workflows/coverage.yml

@@ -67,6 +67,7 @@ jobs:
           SECRET_KEY: ${{ secrets.SECRET_KEY }}
           DJANGO_SETTINGS_MODULE: 'crank.settings'
           REDIS_URL: 'redis://localhost:6379/0'
+          CACHE_TTL: 60
           PYTHON_UNBUFFERED: 1
         run: |
           coverage run -m pytest

.github/workflows/run-tests.yml

@@ -62,6 +62,7 @@ jobs:
           ENV: dev
           SECRET_KEY: '${{ secrets.SECRET_KEY }}'
           REDIS_URL: 'redis://localhost:6379/0'
+          CACHE_TTL: 60
         run: |
           pytest
 

crank/settings/base.py

@@ -158,7 +158,7 @@
 # Set session timeout to 30 minutes
 SESSION_COOKIE_AGE = 1800  # 30 minutes in seconds
 
-CACHE_MIDDLEWARE_SECONDS = 60  # Timeout for cached items in seconds
+CACHE_MIDDLEWARE_SECONDS = int(os.environ["CACHE_TTL"])  # Timeout for cached items in seconds
 REDIS_URL = os.environ["REDIS_URL"]
 # Optional: To use Redis for session storage
 CACHES = {

crank/templatetags/socialapp_cache.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Isaac Adams
+# Licensed under the MIT License. See LICENSE file in the project root for full license information.
+from django import template
+from django.core.cache import cache
+from allauth.socialaccount.models import SocialApp
+from django.conf import settings
+
+register = template.Library()
+
+@register.simple_tag
+def get_cached_social_app(provider):
+    cache_key = f'social_app_{provider}'
+    social_app = cache.get(cache_key)
+    if not social_app:
+        social_app = SocialApp.objects.filter(provider=provider).first()
+        cache.set(cache_key, social_app, timeout=settings.CACHE_MIDDLEWARE_SECONDS)
+    return social_app

crank/tests/views/test_index.py

@@ -5,7 +5,6 @@
 
 from django.contrib.sessions.middleware import SessionMiddleware
 from django.core.serializers import serialize
-from django.http import HttpResponseRedirect
 from django.test import TestCase, Client, RequestFactory, override_settings
 from django.urls import reverse
 from django.utils.html import escape
@@ -18,27 +17,31 @@
 from crank.models.score import Score, ScoreType, ScoreAlgorithm, ScoreAlgorithmWeight
 from crank.views.index import IndexView
 from crank.settings import DEFAULT_ALGORITHM_ID
+from django.core.cache import cache
 
 
-@override_settings(CACHES={'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}})
+@override_settings(CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}})
 class IndexViewTests(TestCase):
 
     def setUp(self):
+        cache.clear()
         self.factory = RequestFactory()
         self.client = Client()
         self.view = IndexView.as_view()
         self.index_url = reverse('index')  # replace 'index' with the actual name of the IndexView in your urls.py
+        ScoreAlgorithm.objects.create(id=DEFAULT_ALGORITHM_ID, name='Test Algorithm',
+                                                          description_content='test.md', status=1)
+        self.algorithms = ScoreAlgorithm.objects.filter(status=1)
+        cache.set('algorithm_object_list', self.algorithms)  # Set the cache
 
-        # Create a SocialApp object for testing
-        self.social_app = SocialApp.objects.create(
+        social_app = SocialApp.objects.create(
             provider='google',
             name='Google',
             client_id='test',
             secret='test',
         )
-        self.score_algorithm = ScoreAlgorithm.objects.create(id=DEFAULT_ALGORITHM_ID, name='Test Algorithm',
-                                                             description_content='test.md')
-        self.social_app.sites.add(Site.objects.get_current())
+        social_app.sites.add(Site.objects.get_current())
+        cache.set('social_app_google', social_app)  # Set the cache
 
     def setup_scores(self):
         # Create some test data
@@ -52,7 +55,7 @@ def setup_scores(self):
         )
 
         score_type = ScoreType.objects.create(id=1, name='Test Score Type')
-        ScoreAlgorithmWeight.objects.create(algorithm_id=self.score_algorithm.id,
+        ScoreAlgorithmWeight.objects.create(algorithm_id=DEFAULT_ALGORITHM_ID,
                                             type_id=score_type.id, weight=1.0)
         Score.objects.create(source_id=self.organization1.id, target_id=self.organization1.id, score=5.0,
                              type_id=score_type.id)
@@ -165,7 +168,7 @@ def test_index_get_queryset(self):
         serialized_org = json.loads(serialize('json', [self.organization1]))[0]['fields']
         self.assertOrgValues(serialized_org, queryset[0], True)
 
-    @patch('crank.views.index.Organization.objects.filter')
+    @patch('crank.models.organization.Organization.objects.filter')
     def test_index_view_organization_does_not_exist(self, mock_filter):
         # Mock the filter method to raise Organization.DoesNotExist
         mock_filter.side_effect = Organization.DoesNotExist

crank/views/index.py

@@ -4,22 +4,15 @@
 
 import markdown
 from django.db import connection
-from django.http import JsonResponse
-from django.shortcuts import redirect
-from django.urls import reverse
 from django.views import generic
 from django.core.cache import cache
 from django.conf import settings
-
-
-from crank.models.organization import Organization
 from crank.models.score import ScoreAlgorithm
 from crank.settings.base import CONTENT_DIR, DEFAULT_ALGORITHM_ID
 from crank.forms.organization_filter import OrganizationFilterForm
 
 
 class IndexView(generic.ListView):
-    algorithm_cache = {}
     template_name = "crank/index.html"
     context_object_name = "top_organization_list"
     paginate_by = 15
@@ -32,21 +25,27 @@ def __init__(self):
         self.algorithm = None
         self.error = None
         self.accelerated_vesting = None
+        cache_key = 'algorithm_object_list'
+        self.algorithms = cache.get(cache_key)
+        if not self.algorithms:
+            self.algorithms = ScoreAlgorithm.objects.filter(status=1)
+            cache.set(cache_key, self.algorithms, timeout=settings.CACHE_MIDDLEWARE_SECONDS)
 
     def _check_algorithm_id(self):
         if not self.algorithm:
             if 'algorithm_id' in self.kwargs:
-                self.algorithm_id = self.kwargs['algorithm_id']
-            if self.algorithm_id in IndexView.algorithm_cache.keys():
-                self.algorithm = IndexView.algorithm_cache[self.algorithm_id]
-                return
-            if not ScoreAlgorithm.objects.filter(id=self.algorithm_id).exists():
-                self.algorithm_id = DEFAULT_ALGORITHM_ID
-                self.request.session["algorithm_id"] = self.algorithm_id
+                self.algorithm_id = int(self.kwargs['algorithm_id'])
+
+            if self.algorithm_id:
+                self.algorithm = self.algorithms.filter(id=int(self.algorithm_id)).first()
+                if self.algorithm:
+                    return
+
+            self.algorithm_id = DEFAULT_ALGORITHM_ID
+            self.request.session["algorithm_id"] = self.algorithm_id
             try:
                 if not self.algorithm:
-                    self.algorithm = ScoreAlgorithm.objects.get(id=self.algorithm_id)
-                    self.algorithm_cache[self.algorithm_id] = self.algorithm
+                    self.algorithm = self.algorithms.filter(id=self.algorithm_id).first()
             except ScoreAlgorithm.DoesNotExist:
                 pass  # we will handle empty algorithms by returning an empty object list
 
@@ -59,7 +58,8 @@ def post(self, request, *args, **kwargs):
     def get_queryset(self):
         # if no algorithm_id in the URL, check for one in the session
         if not self.algorithm_id:
-            self.algorithm_id = self.request.session.get("algorithm_id")
+            self.algorithm_id = int(self.request.session.get(
+                "algorithm_id")) if "algorithm_id" in self.request.session else DEFAULT_ALGORITHM_ID
 
         self.accelerated_vesting = self.request.session.get('accelerated_vesting', False)
 
@@ -112,31 +112,31 @@ def fetch_results():
         self.object_list = cache.get_or_set(cache_key, fetch_results, timeout=settings.CACHE_MIDDLEWARE_SECONDS)
         return self.object_list
 
-
     def get_context_data(self, **kwargs):
         if kwargs is None:
             kwargs = {}
         context = super().get_context_data(**kwargs)
         context['algorithm'] = self.get_algorithm_details()
-        context['all_algorithms'] = ScoreAlgorithm.objects.filter(status=1)
+        context['all_algorithms'] = self.algorithms.filter(status=1)
         context['form'] = OrganizationFilterForm(
             initial={'accelerated_vesting': self.request.session.get('accelerated_vesting')}, request=self.request)
 
-        # Serialize the organization data using JsonResponse
         context['top_organization_list'] = list(self.object_list)
 
         return context
 
     def get_algorithm_details(self):
         if self.error:
             return None
-        self.algorithm_id = self.request.session.get("algorithm_id")
 
         self._check_algorithm_id()
-        if not hasattr(self.algorithm, 'html_description_content'):
-            file_path = os.path.join(CONTENT_DIR, self.algorithm.description_content)
-            with open(file_path, 'r') as file:
-                md_content = file.read()
-            html_content = markdown.markdown(md_content)
-            self.algorithm.html_description_content = html_content
-        return self.algorithm
+        if self.algorithm and not hasattr(self.algorithm, 'html_description_content'):
+            def get_html_content():
+                file_path = os.path.join(CONTENT_DIR, self.algorithm.description_content)
+                with open(file_path, 'r') as file:
+                    md_content = file.read()
+                return markdown.markdown(md_content)
+
+            self.algorithm.html_description_content = cache.get_or_set(
+                f'algorithm_{self.algorithm_id}_description', get_html_content(), timeout=settings.CACHE_MIDDLEWARE_SECONDS)
+        return self.algorithm

k8s/deployment.yml

@@ -86,6 +86,8 @@ spec:
               value: "prod"
             - name: REDIS_URL
               value: "redis://redis:6379/0"
+            - name: CACHE_TTL
+              value: "60"
             - name: PYTHONUNBUFFERED
               value: "1"
           envFrom:

templates/base.html

@@ -3,6 +3,7 @@
 {% load static %}
 {% load socialaccount %}
 {% load manifest %}
+{% load socialapp_cache %}
 <!DOCTYPE html>
 <html lang="en" data-bs-theme="dark">
 <head>
@@ -34,10 +35,13 @@
                     <button class="btn btn-secondary btn-sm" type="submit">Logout {{ user.username }}</button>
                 </form>
             {% else %}
-                <a class="btn btn-light btn-sm" href="{% provider_login_url 'google' %}">
-                    <i class="fab fa-google"></i>
-                    Login with Google
-                </a>
+                {% get_cached_social_app 'google' as social_app %}
+                {% if social_app %}
+                    <a class="btn btn-light btn-sm" href="{% provider_login_url 'google' %}">
+                        <i class="fab fa-google"></i>
+                        Login with Google
+                    </a>
+                {% endif %}
             {% endif %}
         </div>
     </nav>