Merge pull request #639 from iurisilvio/lazy-settings-itersize

Fix #638 Lazy access to Django settings.

changelog.d/638.bugfix

@@ -0,0 +1 @@
+Access `django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE` and `django_redis.client.herd.CACHE_HERD_TIMEOUT` in runtime to not read Django settings in import time.

django_redis/cache.py

@@ -9,8 +9,6 @@
 
 from .exceptions import ConnectionInterrupted
 
-DJANGO_REDIS_SCAN_ITERSIZE = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10)
-
 CONNECTION_INTERRUPTED = object()
 
 
@@ -45,6 +43,9 @@ def __init__(self, server: str, params: Dict[str, Any]) -> None:
         super().__init__(params)
         self._server = server
         self._params = params
+        self._default_scan_itersize = getattr(
+            settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10
+        )
 
         options = params.get("OPTIONS", {})
         self._client_cls = options.get(
@@ -105,7 +106,7 @@ def delete(self, *args, **kwargs):
 
     @omit_exception
     def delete_pattern(self, *args, **kwargs):
-        kwargs["itersize"] = kwargs.get("itersize", DJANGO_REDIS_SCAN_ITERSIZE)
+        kwargs.setdefault("itersize", self._default_scan_itersize)
         return self.client.delete_pattern(*args, **kwargs)
 
     @omit_exception

django_redis/client/herd.py

@@ -21,22 +21,20 @@ class Marker:
     pass
 
 
-CACHE_HERD_TIMEOUT = getattr(settings, "CACHE_HERD_TIMEOUT", 60)
-
-
-def _is_expired(x):
-    if x >= CACHE_HERD_TIMEOUT:
+def _is_expired(x, herd_timeout: int) -> bool:
+    if x >= herd_timeout:
         return True
-    val = x + random.randint(1, CACHE_HERD_TIMEOUT)
+    val = x + random.randint(1, herd_timeout)
 
-    if val >= CACHE_HERD_TIMEOUT:
+    if val >= herd_timeout:
         return True
     return False
 
 
 class HerdClient(DefaultClient):
     def __init__(self, *args, **kwargs):
         self._marker = Marker()
+        self._herd_timeout = getattr(settings, "CACHE_HERD_TIMEOUT", 60)
         super().__init__(*args, **kwargs)
 
     def _pack(self, value, timeout):
@@ -55,7 +53,7 @@ def _unpack(self, value):
         now = int(time.time())
         if herd_timeout < now:
             x = now - herd_timeout
-            return unpacked, _is_expired(x)
+            return unpacked, _is_expired(x, self._herd_timeout)
 
         return unpacked, False
 
@@ -84,7 +82,7 @@ def set(
             )
 
         packed = self._pack(value, timeout)
-        real_timeout = timeout + CACHE_HERD_TIMEOUT
+        real_timeout = timeout + self._herd_timeout
 
         return super().set(
             key, packed, timeout=real_timeout, version=version, client=client, nx=nx

setup.cfg

@@ -100,6 +100,7 @@ REDIS =
 passenv = CI GITHUB*
 commands =
   {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs}
+  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_herd {posargs}
   {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs}
   {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_lz4 {posargs}
   {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_msgpack {posargs}

tests/settings/sqlite_herd.py

@@ -27,3 +27,5 @@
 INSTALLED_APPS = ["django.contrib.sessions"]
 
 USE_TZ = False
+
+CACHE_HERD_TIMEOUT = 2

tests/test_backend.py

@@ -2,21 +2,29 @@
 import threading
 import time
 from datetime import timedelta
-from typing import List, Union, cast
+from typing import Iterable, List, Union, cast
 from unittest.mock import patch
 
 import pytest
 from django.core.cache import caches
+from django.test import override_settings
 from pytest_django.fixtures import SettingsWrapper
 from pytest_mock import MockerFixture
 
-import django_redis.cache
 from django_redis.cache import RedisCache
 from django_redis.client import ShardClient, herd
 from django_redis.serializers.json import JSONSerializer
 from django_redis.serializers.msgpack import MSGPackSerializer
 
-herd.CACHE_HERD_TIMEOUT = 2
+
+@pytest.fixture
+def patch_itersize_setting() -> Iterable[None]:
+    # destroy cache to force recreation with overriden settings
+    del caches["default"]
+    with override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30):
+        yield
+    # destroy cache to force recreation with original settings
+    del caches["default"]
 
 
 class TestDjangoRedisCache:
@@ -199,7 +207,12 @@ def test_set_many(self, cache: RedisCache):
         res = cache.get_many(["a", "b", "c"])
         assert res == {"a": 1, "b": 2, "c": 3}
 
-    def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture):
+    def test_set_call_empty_pipeline(
+        self,
+        cache: RedisCache,
+        mocker: MockerFixture,
+        settings: SettingsWrapper,
+    ):
         if isinstance(cache.client, ShardClient):
             pytest.skip("ShardClient doesn't support get_client")
 
@@ -212,7 +225,7 @@ def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture)
 
         if isinstance(cache.client, herd.HerdClient):
             default_timeout = cache.client._backend.default_timeout
-            herd_timeout = (default_timeout + herd.CACHE_HERD_TIMEOUT) * 1000  # type: ignore # noqa
+            herd_timeout = (default_timeout + settings.CACHE_HERD_TIMEOUT) * 1000
             herd_pack_value = cache.client._pack(value, default_timeout)
             mocked_set.assert_called_once_with(
                 cache.client.make_key(key, version=None),
@@ -495,11 +508,15 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache):
 
     @patch("django_redis.cache.RedisCache.client")
     def test_delete_pattern_with_settings_default_scan_count(
-        self, client_mock, cache: RedisCache
+        self,
+        client_mock,
+        patch_itersize_setting,
+        cache: RedisCache,
+        settings: SettingsWrapper,
     ):
         for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]:
             cache.set(key, "foo")
-        expected_count = django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE
+        expected_count = settings.DJANGO_REDIS_SCAN_ITERSIZE
 
         cache.delete_pattern("*foo-a*")