Add ASGI Django app

api/Pipfile

@@ -38,7 +38,6 @@ drf-spectacular = "*"
 elasticsearch-dsl = "~=7.4"
 future = "~=0.18"
 gevent = "~=22.10"
-gunicorn = "~=20.1"
 hvac = "~=1.0"
 ipaddress = "~=1.0"
 limit = "~=0.2"
@@ -50,8 +49,13 @@ python3-openid = "~=3.2"
 redlock-py = "~=1.0"
 requests-oauthlib = "~=1.3"
 sentry-sdk = "~=1.21"
-wsgi-basic-auth = "~=1.1"
 django-split-settings = "*"
+adrf = "*"
+django-async-redis = "*"
+# Both uvicorn and gunicorn must exist during transition
+# Once we've deployed the app as ASGI we can remove gunicorn
+uvicorn = "*"
+gunicorn = "*"
 
 [requires]
 python_version = "3.11"

api/api/utils/image_proxy/__init__.py

@@ -6,9 +6,11 @@
 from django.http import HttpResponse
 from rest_framework.exceptions import UnsupportedMediaType
 
+import aiohttp
 import django_redis
 import requests
 import sentry_sdk
+from asgiref.sync import sync_to_async
 from sentry_sdk import push_scope, set_context
 
 from api.utils.image_proxy.exception import UpstreamThumbnailException
@@ -58,7 +60,7 @@ def get_request_params_for_extension(
     )
 
 
-def get(
+async def get(
     image_url: str,
     media_identifier: str,
     accept_header: str = "image/*",
@@ -71,9 +73,13 @@ def get(
     """
     logger = parent_logger.getChild("get")
     tallies = django_redis.get_redis_connection("tallies")
+    # For some reason using django_async_redis for the tallies
+    # causes event loop issues. Wrapping the synchronous client's
+    # ``incr`` method prevents issues with using it in an async context
+    incr = sync_to_async(tallies.incr)
     month = get_monthly_timestamp()
 
-    image_extension = get_image_extension(image_url, media_identifier)
+    image_extension = await get_image_extension(image_url, media_identifier)
 
     headers = {"Accept": accept_header} | HEADERS
 
@@ -89,53 +95,57 @@ def get(
         is_compressed,
     )
 
-    try:
-        upstream_response = requests.get(
-            upstream_url,
-            timeout=15,
-            params=params,
-            headers=headers,
-        )
-        tallies.incr(f"thumbnail_response_code:{month}:{upstream_response.status_code}")
-        tallies.incr(
-            f"thumbnail_response_code_by_domain:{domain}:"
-            f"{month}:{upstream_response.status_code}"
-        )
-        upstream_response.raise_for_status()
-    except Exception as exc:
-        exception_name = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
-        key = f"thumbnail_error:{exception_name}:{domain}:{month}"
-        count = tallies.incr(key)
-        if count <= settings.THUMBNAIL_ERROR_INITIAL_ALERT_THRESHOLD or (
-            count % settings.THUMBNAIL_ERROR_REPEATED_ALERT_FREQUENCY == 0
-        ):
-            with push_scope() as scope:
-                set_context(
-                    "upstream_url",
-                    {
-                        "url": upstream_url,
-                        "params": params,
-                        "headers": headers,
-                    },
-                )
-                scope.set_tag(
-                    "occurrences", settings.THUMBNAIL_ERROR_REPEATED_ALERT_FREQUENCY
-                )
-                sentry_sdk.capture_exception(exc)
-        if isinstance(exc, requests.exceptions.HTTPError):
-            tallies.incr(
-                f"thumbnail_http_error:{domain}:{month}:{exc.response.status_code}:{exc.response.text}"
+    async with aiohttp.ClientSession(
+        headers=headers, timeout=aiohttp.ClientTimeout(total=15)
+    ) as client:
+        try:
+            upstream_response = await client.get(
+                upstream_url,
+                params=params,
             )
-        raise UpstreamThumbnailException(f"Failed to render thumbnail. {exc}")
 
-    res_status = upstream_response.status_code
+            await incr(f"thumbnail_response_code:{month}:{upstream_response.status}")
+            await incr(
+                f"thumbnail_response_code_by_domain:{domain}:"
+                f"{month}:{upstream_response.status}"
+            )
+            upstream_response.raise_for_status()
+            body = await upstream_response.read()
+        except Exception as exc:
+            raise exc
+            exception_name = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
+            key = f"thumbnail_error:{exception_name}:{domain}:{month}"
+            count = await incr(key)
+            if count <= settings.THUMBNAIL_ERROR_INITIAL_ALERT_THRESHOLD or (
+                count % settings.THUMBNAIL_ERROR_REPEATED_ALERT_FREQUENCY == 0
+            ):
+                with push_scope() as scope:
+                    set_context(
+                        "upstream_url",
+                        {
+                            "url": upstream_url,
+                            "params": params,
+                            "headers": headers,
+                        },
+                    )
+                    scope.set_tag(
+                        "occurrences", settings.THUMBNAIL_ERROR_REPEATED_ALERT_FREQUENCY
+                    )
+                    sentry_sdk.capture_exception(exc)
+            if isinstance(exc, requests.exceptions.HTTPError):
+                await incr(
+                    f"thumbnail_http_error:{domain}:{month}:{exc.response.status_code}:{exc.response.text}"
+                )
+            raise UpstreamThumbnailException(f"Failed to render thumbnail. {exc}")
+
+    res_status = upstream_response.status
     content_type = upstream_response.headers.get("Content-Type")
     logger.debug(
         f"Image proxy response status: {res_status}, content-type: {content_type}"
     )
 
     return HttpResponse(
-        upstream_response.content,
+        body,
         status=res_status,
         content_type=content_type,
     )

api/api/utils/image_proxy/extension.py

@@ -1,43 +1,46 @@
 from os.path import splitext
 from urllib.parse import urlparse
 
-import django_redis
-import requests
+import aiohttp
+import django_async_redis
 import sentry_sdk
 
 from api.utils.image_proxy.exception import UpstreamThumbnailException
 
 
-def get_image_extension(image_url: str, media_identifier: str) -> str | None:
-    cache = django_redis.get_redis_connection("default")
+async def get_image_extension(image_url: str, media_identifier: str) -> str | None:
+    cache = await django_async_redis.get_redis_connection("adefault")
     key = f"media:{media_identifier}:thumb_type"
 
     ext = _get_file_extension_from_url(image_url)
 
     if not ext:
         # If the extension is not present in the URL, try to get it from the redis cache
-        ext = cache.get(key)
+        ext = await cache.get(key)
         ext = ext.decode("utf-8") if ext else None
 
     if not ext:
         # If the extension is still not present, try getting it from the content type
-        try:
-            response = requests.head(image_url, timeout=10)
-            response.raise_for_status()
-        except Exception as exc:
-            sentry_sdk.capture_exception(exc)
-            raise UpstreamThumbnailException(
-                "Failed to render thumbnail due to inability to check media "
-                f"type. {exc}"
-            )
-        else:
-            if response.headers and "Content-Type" in response.headers:
-                content_type = response.headers["Content-Type"]
-                ext = _get_file_extension_from_content_type(content_type)
+        async with aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(total=10)
+        ) as client:
+            try:
+                response = await client.head(image_url, timeout=10)
+                response.raise_for_status()
+            except Exception as exc:
+                sentry_sdk.capture_exception(exc)
+                raise UpstreamThumbnailException(
+                    "Failed to render thumbnail due to inability to check media "
+                    f"type. {exc}"
+                )
             else:
-                ext = None
+                if response.headers and "Content-Type" in response.headers:
+                    content_type = response.headers["Content-Type"]
+                    ext = _get_file_extension_from_content_type(content_type)
+                else:
+                    ext = None
 
-            cache.set(key, ext if ext else "unknown")
+                await cache.set(key, ext if ext else "unknown")
     return ext
 
 

api/api/views/audio_views.py

@@ -24,7 +24,7 @@
 )
 from api.serializers.media_serializers import MediaThumbnailRequestSerializer
 from api.utils.throttle import AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle
-from api.views.media_views import MediaViewSet
+from api.views.media_views import AsyncMediaView, MediaViewSet
 
 
 @extend_schema(tags=["audio"])
@@ -48,32 +48,6 @@ def get_queryset(self):
 
     # Extra actions
 
-    @thumbnail
-    @action(
-        detail=True,
-        url_path="thumb",
-        url_name="thumb",
-        serializer_class=MediaThumbnailRequestSerializer,
-        throttle_classes=[AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle],
-    )
-    def thumbnail(self, request, *_, **__):
-        """
-        Retrieve the scaled down and compressed thumbnail of the artwork of an
-        audio track or its audio set.
-        """
-
-        audio = self.get_object()
-
-        image_url = None
-        if audio_thumbnail := audio.thumbnail:
-            image_url = audio_thumbnail
-        elif audio.audio_set and (audio_thumbnail := audio.audio_set.thumbnail):
-            image_url = audio_thumbnail
-        if not image_url:
-            raise NotFound("Could not find artwork.")
-
-        return super().thumbnail(request, audio, image_url)
-
     @waveform
     @action(
         detail=True,
@@ -114,3 +88,33 @@ def report(self, request, identifier):
         """
 
         return super().report(request, identifier)
+
+
+class AsyncAudioView(AsyncMediaView):
+    model_class = Audio
+
+    @thumbnail
+    @action(
+        detail=True,
+        url_path="thumb",
+        url_name="thumb",
+        serializer_class=MediaThumbnailRequestSerializer,
+        throttle_classes=[AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle],
+    )
+    async def thumbnail(self, request, *_, **__):
+        """
+        Retrieve the scaled down and compressed thumbnail of the artwork of an
+        audio track or its audio set.
+        """
+
+        audio = await self.aget_object()
+
+        image_url = None
+        if audio_thumbnail := audio.thumbnail:
+            image_url = audio_thumbnail
+        elif audio.audio_set and (audio_thumbnail := audio.audio_set.thumbnail):
+            image_url = audio_thumbnail
+        if not image_url:
+            raise NotFound("Could not find artwork.")
+
+        return await super().get_thumbnail(request, audio, image_url)

api/api/views/image_views.py

@@ -34,7 +34,7 @@
 from api.serializers.media_serializers import MediaThumbnailRequestSerializer
 from api.utils.throttle import AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle
 from api.utils.watermark import watermark
-from api.views.media_views import MediaViewSet
+from api.views.media_views import AsyncMediaView, MediaViewSet
 
 
 @extend_schema(tags=["images"])
@@ -99,26 +99,6 @@ def oembed(self, request, *_, **__):
         serializer = self.get_serializer(image, context=context)
         return Response(data=serializer.data)
 
-    @thumbnail
-    @action(
-        detail=True,
-        url_path="thumb",
-        url_name="thumb",
-        serializer_class=MediaThumbnailRequestSerializer,
-        throttle_classes=[AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle],
-    )
-    def thumbnail(self, request, *_, **__):
-        """Retrieve the scaled down and compressed thumbnail of the image."""
-
-        image = self.get_object()
-        image_url = image.url
-        # Hotfix to use thumbnails for SMK images
-        # TODO: Remove when small thumbnail issues are resolved
-        if "iip.smk.dk" in image_url and image.thumbnail:
-            image_url = image.thumbnail
-
-        return super().thumbnail(request, image, image_url)
-
     @watermark_doc
     @action(detail=True, url_path="watermark", url_name="watermark")
     def watermark(self, request, *_, **__):  # noqa: D401
@@ -203,3 +183,27 @@ def _save_wrapper(pil_img, exif_bytes, destination):
             pil_img.save(destination, "jpeg", exif=exif_bytes)
         else:
             pil_img.save(destination, "jpeg")
+
+
+class AsyncImageView(AsyncMediaView):
+    model_class = Image
+
+    @thumbnail
+    @action(
+        detail=True,
+        url_path="thumb",
+        url_name="thumb",
+        serializer_class=MediaThumbnailRequestSerializer,
+        throttle_classes=[AnonThumbnailRateThrottle, OAuth2IdThumbnailRateThrottle],
+    )
+    async def thumbnail(self, request, *_, **__):
+        """Retrieve the scaled down and compressed thumbnail of the image."""
+
+        image = await self.aget_object()
+        image_url = image.url
+        # Hotfix to use thumbnails for SMK images
+        # TODO: Remove when small thumbnail issues are resolved
+        if "iip.smk.dk" in image_url and image.thumbnail:
+            image_url = image.thumbnail
+
+        return await super().get_thumbnail(request, image, image_url)