From 9dc3e3029c48b1e1f492c49df38e19aed0714315 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Fri, 6 Jan 2023 20:27:47 -0300 Subject: [PATCH 01/24] Lazy access to Django settings. --- django_redis/cache.py | 5 ++--- django_redis/client/herd.py | 16 +++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/django_redis/cache.py b/django_redis/cache.py index ebf4d14b..795e5011 100644 --- a/django_redis/cache.py +++ b/django_redis/cache.py @@ -9,8 +9,6 @@ from .exceptions import ConnectionInterrupted -DJANGO_REDIS_SCAN_ITERSIZE = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) - CONNECTION_INTERRUPTED = object() @@ -105,7 +103,8 @@ def delete(self, *args, **kwargs): @omit_exception def delete_pattern(self, *args, **kwargs): - kwargs["itersize"] = kwargs.get("itersize", DJANGO_REDIS_SCAN_ITERSIZE) + if not kwargs.get("itersize"): + kwargs["itersize"] = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) return self.client.delete_pattern(*args, **kwargs) @omit_exception diff --git a/django_redis/client/herd.py b/django_redis/client/herd.py index d5ab941d..555f0d52 100644 --- a/django_redis/client/herd.py +++ b/django_redis/client/herd.py @@ -21,15 +21,12 @@ class Marker: pass -CACHE_HERD_TIMEOUT = getattr(settings, "CACHE_HERD_TIMEOUT", 60) - - -def _is_expired(x): - if x >= CACHE_HERD_TIMEOUT: +def _is_expired(x, herd_timeout): + if x >= herd_timeout: return True - val = x + random.randint(1, CACHE_HERD_TIMEOUT) + val = x + random.randint(1, herd_timeout) - if val >= CACHE_HERD_TIMEOUT: + if val >= herd_timeout: return True return False @@ -37,6 +34,7 @@ def _is_expired(x): class HerdClient(DefaultClient): def __init__(self, *args, **kwargs): self._marker = Marker() + self._herd_timeout = getattr(settings, "CACHE_HERD_TIMEOUT", 60) super().__init__(*args, **kwargs) def _pack(self, value, timeout): @@ -55,7 +53,7 @@ def _unpack(self, value): now = int(time.time()) if herd_timeout < now: x = now - herd_timeout - return unpacked, _is_expired(x) + return unpacked, _is_expired(x, self._herd_timeout) return unpacked, False @@ -84,7 +82,7 @@ def set( ) packed = self._pack(value, timeout) - real_timeout = timeout + CACHE_HERD_TIMEOUT + real_timeout = timeout + self._herd_timeout return super().set( key, packed, timeout=real_timeout, version=version, client=client, nx=nx From 0105fb1e819be269e81f199b911b697d63cfdcff Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Fri, 6 Jan 2023 20:55:17 -0300 Subject: [PATCH 02/24] Fix tests. --- tests/test_backend.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9fdba5a4..c2ea2551 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest +from django.conf import settings from django.core.cache import caches from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture @@ -16,8 +17,6 @@ from django_redis.serializers.json import JSONSerializer from django_redis.serializers.msgpack import MSGPackSerializer -herd.CACHE_HERD_TIMEOUT = 2 - class TestDjangoRedisCache: def test_setnx(self, cache: RedisCache): @@ -199,7 +198,9 @@ def test_set_many(self, cache: RedisCache): res = cache.get_many(["a", "b", "c"]) assert res == {"a": 1, "b": 2, "c": 3} - def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture): + def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper): + settings.CACHE_HERD_TIMEOUT = 2 + if isinstance(cache.client, ShardClient): pytest.skip("ShardClient doesn't support get_client") @@ -212,7 +213,7 @@ def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture) if isinstance(cache.client, herd.HerdClient): default_timeout = cache.client._backend.default_timeout - herd_timeout = (default_timeout + herd.CACHE_HERD_TIMEOUT) * 1000 # type: ignore # noqa + herd_timeout = (default_timeout + settings.CACHE_HERD_TIMEOUT) * 1000 # type: ignore # noqa herd_pack_value = cache.client._pack(value, default_timeout) mocked_set.assert_called_once_with( cache.client.make_key(key, version=None), @@ -495,11 +496,13 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, cache: RedisCache + self, client_mock, cache: RedisCache, settings: SettingsWrapper, ): + settings.DJANGO_REDIS_SCAN_ITERSIZE = 30 + for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") - expected_count = django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE + expected_count = settings.DJANGO_REDIS_SCAN_ITERSIZE cache.delete_pattern("*foo-a*") From b88a718a3d1c5e126d3cd881759f57b2ba1b1b10 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Fri, 6 Jan 2023 20:55:33 -0300 Subject: [PATCH 03/24] Fix tox passenv. I don't know it changed. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e0106810..2aa38cc1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -97,7 +97,7 @@ REDIS = master: redismaster [testenv] -passenv = CI GITHUB* +passenv = CI,GITHUB* commands = {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs} From 7a61ec9db0825d92374a2da3ba5852d3eb8c4177 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Fri, 6 Jan 2023 20:57:45 -0300 Subject: [PATCH 04/24] LInt. --- tests/test_backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index c2ea2551..9594e51d 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -6,12 +6,10 @@ from unittest.mock import patch import pytest -from django.conf import settings from django.core.cache import caches from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture -import django_redis.cache from django_redis.cache import RedisCache from django_redis.client import ShardClient, herd from django_redis.serializers.json import JSONSerializer From dde64b73828cedac6e4b4d1145c08c58627b4289 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Fri, 6 Jan 2023 21:22:28 -0300 Subject: [PATCH 05/24] changelog --- changelog.d/638.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/638.removal diff --git a/changelog.d/638.removal b/changelog.d/638.removal new file mode 100644 index 00000000..909ec48e --- /dev/null +++ b/changelog.d/638.removal @@ -0,0 +1 @@ +Removed `django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE` and `django_redis.client.herd.CACHE_HERD_TIMEOUT` to not access Django settings in import time. \ No newline at end of file From fe99e5c841cfce4b5ac1efefe038c688935d3ea5 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Thu, 19 Jan 2023 11:48:03 -0300 Subject: [PATCH 06/24] revert passenv --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2aa38cc1..e0106810 100644 --- a/setup.cfg +++ b/setup.cfg @@ -97,7 +97,7 @@ REDIS = master: redismaster [testenv] -passenv = CI,GITHUB* +passenv = CI GITHUB* commands = {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs} From 7284dacdd913f0bd4f1707dc2f7a4c83016d16a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 13:42:13 +0000 Subject: [PATCH 07/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_backend.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9594e51d..3206bfce 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -196,7 +196,9 @@ def test_set_many(self, cache: RedisCache): res = cache.get_many(["a", "b", "c"]) assert res == {"a": 1, "b": 2, "c": 3} - def test_set_call_empty_pipeline(self, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper): + def test_set_call_empty_pipeline( + self, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper + ): settings.CACHE_HERD_TIMEOUT = 2 if isinstance(cache.client, ShardClient): @@ -494,7 +496,10 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, cache: RedisCache, settings: SettingsWrapper, + self, + client_mock, + cache: RedisCache, + settings: SettingsWrapper, ): settings.DJANGO_REDIS_SCAN_ITERSIZE = 30 From 0a2ca0d881287114bc77ab91b8607f7b25ad748e Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Thu, 1 Jun 2023 16:20:04 +0100 Subject: [PATCH 08/24] Run herd tests. --- setup.cfg | 1 + tests/conftest.py | 6 ++++++ tests/test_backend.py | 2 -- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index e0106810..3c76f354 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,7 @@ REDIS = passenv = CI GITHUB* commands = {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs} + {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_herd {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_lz4 {posargs} {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_msgpack {posargs} diff --git a/tests/conftest.py b/tests/conftest.py index 59ea7d8d..85e9e57f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,17 @@ from typing import Iterable import pytest +from pytest_django.fixtures import SettingsWrapper from django.core.cache import cache as default_cache from django_redis.cache import BaseCache +@pytest.fixture(autouse=True) +def patch_settings(settings: SettingsWrapper): + settings.CACHE_HERD_TIMEOUT = 2 + + @pytest.fixture def cache() -> Iterable[BaseCache]: yield default_cache diff --git a/tests/test_backend.py b/tests/test_backend.py index 3206bfce..5ddfa1dc 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -199,8 +199,6 @@ def test_set_many(self, cache: RedisCache): def test_set_call_empty_pipeline( self, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper ): - settings.CACHE_HERD_TIMEOUT = 2 - if isinstance(cache.client, ShardClient): pytest.skip("ShardClient doesn't support get_client") From dc91afbcf1018c4b55803f1323a9f5903059261d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 15:22:45 +0000 Subject: [PATCH 09/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 85e9e57f..61ce16cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,8 @@ from typing import Iterable import pytest -from pytest_django.fixtures import SettingsWrapper from django.core.cache import cache as default_cache +from pytest_django.fixtures import SettingsWrapper from django_redis.cache import BaseCache From 3b597f05a2896494e1f2b43b7246599fd54fe335 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Thu, 1 Jun 2023 16:35:11 +0100 Subject: [PATCH 10/24] Fix lint and mypy. --- tests/test_backend.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 5ddfa1dc..4445fd75 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -211,7 +211,7 @@ def test_set_call_empty_pipeline( if isinstance(cache.client, herd.HerdClient): default_timeout = cache.client._backend.default_timeout - herd_timeout = (default_timeout + settings.CACHE_HERD_TIMEOUT) * 1000 # type: ignore # noqa + herd_timeout = (default_timeout + settings.CACHE_HERD_TIMEOUT) * 1000 herd_pack_value = cache.client._pack(value, default_timeout) mocked_set.assert_called_once_with( cache.client.make_key(key, version=None), @@ -494,10 +494,7 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, - client_mock, - cache: RedisCache, - settings: SettingsWrapper, + self, client_mock, cache: RedisCache, settings: SettingsWrapper, ): settings.DJANGO_REDIS_SCAN_ITERSIZE = 30 From db54da5b7c18d7ce8f9ae271f966e3e28bb8ed50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 15:35:42 +0000 Subject: [PATCH 11/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_backend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 4445fd75..e222c7e7 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -494,7 +494,10 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, cache: RedisCache, settings: SettingsWrapper, + self, + client_mock, + cache: RedisCache, + settings: SettingsWrapper, ): settings.DJANGO_REDIS_SCAN_ITERSIZE = 30 From 9c787b56dc44aebc3833b51cac80a04f3d7e08ad Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 12:50:30 +0200 Subject: [PATCH 12/24] Changelog it as bugfix. --- changelog.d/638.bugfix | 1 + changelog.d/638.removal | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/638.bugfix delete mode 100644 changelog.d/638.removal diff --git a/changelog.d/638.bugfix b/changelog.d/638.bugfix new file mode 100644 index 00000000..ade737ce --- /dev/null +++ b/changelog.d/638.bugfix @@ -0,0 +1 @@ +Access `django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE` and `django_redis.client.herd.CACHE_HERD_TIMEOUT` in runtime to not read Django settings in import time. \ No newline at end of file diff --git a/changelog.d/638.removal b/changelog.d/638.removal deleted file mode 100644 index 909ec48e..00000000 --- a/changelog.d/638.removal +++ /dev/null @@ -1 +0,0 @@ -Removed `django_redis.cache.DJANGO_REDIS_SCAN_ITERSIZE` and `django_redis.client.herd.CACHE_HERD_TIMEOUT` to not access Django settings in import time. \ No newline at end of file From 113c294da3247e14fe0b47ed698f8c67db63b2d9 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 12:51:10 +0200 Subject: [PATCH 13/24] Persist itersize on RedisCache init. --- django_redis/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_redis/cache.py b/django_redis/cache.py index 795e5011..8a0e81b7 100644 --- a/django_redis/cache.py +++ b/django_redis/cache.py @@ -43,6 +43,7 @@ def __init__(self, server: str, params: Dict[str, Any]) -> None: super().__init__(params) self._server = server self._params = params + self._default_scan_itersize = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) options = params.get("OPTIONS", {}) self._client_cls = options.get( @@ -103,8 +104,7 @@ def delete(self, *args, **kwargs): @omit_exception def delete_pattern(self, *args, **kwargs): - if not kwargs.get("itersize"): - kwargs["itersize"] = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) + kwargs.setdefault("itersize", self._default_scan_itersize) return self.client.delete_pattern(*args, **kwargs) @omit_exception From cfdd96f210e70cf6475596e61a8e901114437a76 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 13:04:46 +0200 Subject: [PATCH 14/24] Use override_settings instead of pytest test settings. --- tests/test_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index e222c7e7..8fa06e0a 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -7,6 +7,7 @@ import pytest from django.core.cache import caches +from django.test import override_settings from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture @@ -493,14 +494,13 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): client_mock.delete_pattern.assert_called_once_with("*foo-a*", itersize=2) @patch("django_redis.cache.RedisCache.client") + @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( self, client_mock, cache: RedisCache, settings: SettingsWrapper, ): - settings.DJANGO_REDIS_SCAN_ITERSIZE = 30 - for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") expected_count = settings.DJANGO_REDIS_SCAN_ITERSIZE From 5e5d09515a147d5e97b76ef85538de4a6cd01e91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Jun 2023 11:05:14 +0000 Subject: [PATCH 15/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_redis/cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_redis/cache.py b/django_redis/cache.py index 8a0e81b7..0b4d5890 100644 --- a/django_redis/cache.py +++ b/django_redis/cache.py @@ -43,7 +43,9 @@ def __init__(self, server: str, params: Dict[str, Any]) -> None: super().__init__(params) self._server = server self._params = params - self._default_scan_itersize = getattr(settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10) + self._default_scan_itersize = getattr( + settings, "DJANGO_REDIS_SCAN_ITERSIZE", 10 + ) options = params.get("OPTIONS", {}) self._client_cls = options.get( From 06a7cae105b85e5f17fd95de4acd3ac995189dee Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 19:43:44 +0200 Subject: [PATCH 16/24] Use local fixture to patch herd settings, --- tests/conftest.py | 6 ------ tests/test_backend.py | 35 +++++++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 61ce16cf..59ea7d8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,16 +2,10 @@ import pytest from django.core.cache import cache as default_cache -from pytest_django.fixtures import SettingsWrapper from django_redis.cache import BaseCache -@pytest.fixture(autouse=True) -def patch_settings(settings: SettingsWrapper): - settings.CACHE_HERD_TIMEOUT = 2 - - @pytest.fixture def cache() -> Iterable[BaseCache]: yield default_cache diff --git a/tests/test_backend.py b/tests/test_backend.py index 8fa06e0a..9474f642 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -17,6 +17,20 @@ from django_redis.serializers.msgpack import MSGPackSerializer +@pytest.fixture +def patch_herd_settings(): + default_cache = caches["default"] + if not isinstance(default_cache.client, herd.HerdClient): + yield + + # destroy cache to force recreation with updated settings + del caches["default"] + with override_settings(CACHE_HERD_TIMEOUT=2): + yield + # destroy cache force recreation with original settings + del caches["default"] + + class TestDjangoRedisCache: def test_setnx(self, cache: RedisCache): # we should ensure there is no test_key_nx in redis @@ -36,7 +50,7 @@ def test_setnx(self, cache: RedisCache): res = cache.get("test_key_nx") assert res is None - def test_setnx_timeout(self, cache: RedisCache): + def test_setnx_timeout(self, patch_herd_settings, cache: RedisCache): # test that timeout still works for nx=True res = cache.set("test_key_nx", 1, timeout=2, nx=True) assert res is True @@ -115,7 +129,7 @@ def test_save_float(self, cache: RedisCache): assert isinstance(res, float) assert res == float_val - def test_timeout(self, cache: RedisCache): + def test_timeout(self, patch_herd_settings, cache: RedisCache): cache.set("test_key", 222, timeout=3) time.sleep(4) @@ -127,7 +141,7 @@ def test_timeout_0(self, cache: RedisCache): res = cache.get("test_key") assert res is None - def test_timeout_parameter_as_positional_argument(self, cache: RedisCache): + def test_timeout_parameter_as_positional_argument(self, patch_herd_settings, cache: RedisCache): cache.set("test_key", 222, -1) res = cache.get("test_key") assert res is None @@ -198,7 +212,11 @@ def test_set_many(self, cache: RedisCache): assert res == {"a": 1, "b": 2, "c": 3} def test_set_call_empty_pipeline( - self, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper + self, + patch_herd_settings, + cache: RedisCache, + mocker: MockerFixture, + settings: SettingsWrapper, ): if isinstance(cache.client, ShardClient): pytest.skip("ShardClient doesn't support get_client") @@ -496,10 +514,7 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( - self, - client_mock, - cache: RedisCache, - settings: SettingsWrapper, + self, client_mock, patch_herd_settings, cache: RedisCache, settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") @@ -521,7 +536,7 @@ def test_close_client(self, cache: RedisCache, mocker: MockerFixture): cache.close() assert mock.called - def test_ttl(self, cache: RedisCache): + def test_ttl(self, patch_herd_settings, cache: RedisCache): cache.set("foo", "bar", 10) ttl = cache.ttl("foo") @@ -728,7 +743,7 @@ def test_touch_zero_timeout(self, cache: RedisCache): res = cache.get("test_key") assert res is None - def test_touch_positive_timeout(self, cache: RedisCache): + def test_touch_positive_timeout(self, patch_herd_settings, cache: RedisCache): cache.set("test_key", 222, timeout=10) assert cache.touch("test_key", 2) is True From 9992d6bf5635ebe8b558a19ca6b9b034b3058204 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Jun 2023 17:44:43 +0000 Subject: [PATCH 17/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_backend.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9474f642..903b0112 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -141,7 +141,9 @@ def test_timeout_0(self, cache: RedisCache): res = cache.get("test_key") assert res is None - def test_timeout_parameter_as_positional_argument(self, patch_herd_settings, cache: RedisCache): + def test_timeout_parameter_as_positional_argument( + self, patch_herd_settings, cache: RedisCache + ): cache.set("test_key", 222, -1) res = cache.get("test_key") assert res is None @@ -514,7 +516,11 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, patch_herd_settings, cache: RedisCache, settings: SettingsWrapper, + self, + client_mock, + patch_herd_settings, + cache: RedisCache, + settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") From e486bb6ca5fcac66e9fe39b7d18813c1db33ead2 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 19:48:31 +0200 Subject: [PATCH 18/24] Only one yield. --- tests/test_backend.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 903b0112..85c0735b 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -20,16 +20,16 @@ @pytest.fixture def patch_herd_settings(): default_cache = caches["default"] - if not isinstance(default_cache.client, herd.HerdClient): + if isinstance(default_cache.client, herd.HerdClient): + # destroy cache to force recreation with updated settings + del caches["default"] + with override_settings(CACHE_HERD_TIMEOUT=2): + yield + # destroy cache force recreation with original settings + del caches["default"] + else: yield - # destroy cache to force recreation with updated settings - del caches["default"] - with override_settings(CACHE_HERD_TIMEOUT=2): - yield - # destroy cache force recreation with original settings - del caches["default"] - class TestDjangoRedisCache: def test_setnx(self, cache: RedisCache): From 36a54cc886d93d5486d1ab11dbfa4d0351228398 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 20:31:11 +0200 Subject: [PATCH 19/24] Cleanup. --- tests/settings/sqlite_herd.py | 2 ++ tests/test_backend.py | 33 ++++++--------------------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/tests/settings/sqlite_herd.py b/tests/settings/sqlite_herd.py index 7cfb7af4..9bac3a4c 100644 --- a/tests/settings/sqlite_herd.py +++ b/tests/settings/sqlite_herd.py @@ -27,3 +27,5 @@ INSTALLED_APPS = ["django.contrib.sessions"] USE_TZ = False + +CACHE_HERD_TIMEOUT = 2 diff --git a/tests/test_backend.py b/tests/test_backend.py index 85c0735b..dfedd24d 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -17,20 +17,6 @@ from django_redis.serializers.msgpack import MSGPackSerializer -@pytest.fixture -def patch_herd_settings(): - default_cache = caches["default"] - if isinstance(default_cache.client, herd.HerdClient): - # destroy cache to force recreation with updated settings - del caches["default"] - with override_settings(CACHE_HERD_TIMEOUT=2): - yield - # destroy cache force recreation with original settings - del caches["default"] - else: - yield - - class TestDjangoRedisCache: def test_setnx(self, cache: RedisCache): # we should ensure there is no test_key_nx in redis @@ -50,7 +36,7 @@ def test_setnx(self, cache: RedisCache): res = cache.get("test_key_nx") assert res is None - def test_setnx_timeout(self, patch_herd_settings, cache: RedisCache): + def test_setnx_timeout(self, cache: RedisCache): # test that timeout still works for nx=True res = cache.set("test_key_nx", 1, timeout=2, nx=True) assert res is True @@ -129,7 +115,7 @@ def test_save_float(self, cache: RedisCache): assert isinstance(res, float) assert res == float_val - def test_timeout(self, patch_herd_settings, cache: RedisCache): + def test_timeout(self, cache: RedisCache): cache.set("test_key", 222, timeout=3) time.sleep(4) @@ -141,9 +127,7 @@ def test_timeout_0(self, cache: RedisCache): res = cache.get("test_key") assert res is None - def test_timeout_parameter_as_positional_argument( - self, patch_herd_settings, cache: RedisCache - ): + def test_timeout_parameter_as_positional_argument(self, cache: RedisCache): cache.set("test_key", 222, -1) res = cache.get("test_key") assert res is None @@ -215,7 +199,6 @@ def test_set_many(self, cache: RedisCache): def test_set_call_empty_pipeline( self, - patch_herd_settings, cache: RedisCache, mocker: MockerFixture, settings: SettingsWrapper, @@ -516,11 +499,7 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( - self, - client_mock, - patch_herd_settings, - cache: RedisCache, - settings: SettingsWrapper, + self, client_mock, cache: RedisCache, settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") @@ -542,7 +521,7 @@ def test_close_client(self, cache: RedisCache, mocker: MockerFixture): cache.close() assert mock.called - def test_ttl(self, patch_herd_settings, cache: RedisCache): + def test_ttl(self, cache: RedisCache): cache.set("foo", "bar", 10) ttl = cache.ttl("foo") @@ -749,7 +728,7 @@ def test_touch_zero_timeout(self, cache: RedisCache): res = cache.get("test_key") assert res is None - def test_touch_positive_timeout(self, patch_herd_settings, cache: RedisCache): + def test_touch_positive_timeout(self, cache: RedisCache): cache.set("test_key", 222, timeout=10) assert cache.touch("test_key", 2) is True From 60500d11bf774e12e5d60c658cc91cfc8dbc9af9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Jun 2023 18:35:37 +0000 Subject: [PATCH 20/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_backend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index dfedd24d..440d1fef 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -499,7 +499,10 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, cache: RedisCache, settings: SettingsWrapper, + self, + client_mock, + cache: RedisCache, + settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") From 418c89ff05a2efbf9c8498959454a6f7c1bd81ac Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 21:14:41 +0200 Subject: [PATCH 21/24] Fixture to patch itersize. --- tests/test_backend.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 440d1fef..35f6f9ea 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -17,6 +17,16 @@ from django_redis.serializers.msgpack import MSGPackSerializer +@pytest.fixture +def patch_itersize(): + # destroy cache to force recreation with overriden settings + del caches["default"] + with override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30): + yield + # destroy cache to force recreation with original settings + del caches["default"] + + class TestDjangoRedisCache: def test_setnx(self, cache: RedisCache): # we should ensure there is no test_key_nx in redis @@ -497,12 +507,8 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): client_mock.delete_pattern.assert_called_once_with("*foo-a*", itersize=2) @patch("django_redis.cache.RedisCache.client") - @override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30) def test_delete_pattern_with_settings_default_scan_count( - self, - client_mock, - cache: RedisCache, - settings: SettingsWrapper, + self, client_mock, patch_itersize, cache: RedisCache, settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") From 84209b506c7e79f3334227b759795838e95a3d5b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Jun 2023 19:15:41 +0000 Subject: [PATCH 22/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 35f6f9ea..9766b7fa 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -508,7 +508,11 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): @patch("django_redis.cache.RedisCache.client") def test_delete_pattern_with_settings_default_scan_count( - self, client_mock, patch_itersize, cache: RedisCache, settings: SettingsWrapper, + self, + client_mock, + patch_itersize, + cache: RedisCache, + settings: SettingsWrapper, ): for key in ["foo-aa", "foo-ab", "foo-bb", "foo-bc"]: cache.set(key, "foo") From 0af03b47233f580de678961f7ad05a6c84437728 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 21:36:48 +0200 Subject: [PATCH 23/24] Trying to make mypy happy. --- django_redis/client/herd.py | 2 +- tests/test_backend.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_redis/client/herd.py b/django_redis/client/herd.py index 555f0d52..0c52480f 100644 --- a/django_redis/client/herd.py +++ b/django_redis/client/herd.py @@ -21,7 +21,7 @@ class Marker: pass -def _is_expired(x, herd_timeout): +def _is_expired(x, herd_timeout: int) -> bool: if x >= herd_timeout: return True val = x + random.randint(1, herd_timeout) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9766b7fa..8c730c76 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -18,7 +18,7 @@ @pytest.fixture -def patch_itersize(): +def patch_itersize_setting() -> Iterable[None]: # destroy cache to force recreation with overriden settings del caches["default"] with override_settings(DJANGO_REDIS_SCAN_ITERSIZE=30): @@ -510,7 +510,7 @@ def test_delete_pattern_with_custom_count(self, client_mock, cache: RedisCache): def test_delete_pattern_with_settings_default_scan_count( self, client_mock, - patch_itersize, + patch_itersize_setting, cache: RedisCache, settings: SettingsWrapper, ): From e7ee9815cad68f449a999bb6da7b8678b801c9c7 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Sun, 11 Jun 2023 21:39:54 +0200 Subject: [PATCH 24/24] Import iterable. --- tests/test_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 8c730c76..7f4beb70 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -2,7 +2,7 @@ import threading import time from datetime import timedelta -from typing import List, Union, cast +from typing import Iterable, List, Union, cast from unittest.mock import patch import pytest