[WIP] External sharded cache

changelog.d/11046.feature

@@ -0,0 +1 @@
+Add new external sharded Redis cache. Contributed by Nick @ Beeper.

pyproject.toml

@@ -178,6 +178,7 @@ jaeger-client = { version = ">=4.0.0", optional = true }
 pyjwt = { version = ">=1.6.4", optional = true }
 txredisapi = { version = ">=1.4.7", optional = true }
 hiredis = { version = "*", optional = true }
+jump-consistent-hash = { version = ">=3.2.0", optional = true }
 Pympler = { version = "*", optional = true }
 parameterized = { version = ">=0.7.4", optional = true }
 idna = { version = ">=2.5", optional = true }
@@ -199,7 +200,7 @@ opentracing = ["jaeger-client", "opentracing"]
 jwt = ["pyjwt"]
 # hiredis is not a *strict* dependency, but it makes things much faster.
 # (if it is not installed, we fall back to slow code.)
-redis = ["txredisapi", "hiredis"]
+redis = ["txredisapi", "hiredis", "jump-consistent-hash"]
 # Required to use experimental `caches.track_memory_usage` config option.
 cache_memory = ["pympler"]
 test = ["parameterized", "idna"]
@@ -233,7 +234,7 @@ all = [
     # jwt
     "pyjwt",
     # redis
-    "txredisapi", "hiredis",
+    "txredisapi", "hiredis", "jump-consistent-hash",
     # cache_memory
     "pympler",
     # omitted:

stubs/jump.pyi

@@ -0,0 +1,15 @@
+# Copyright 2022 Beeper
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def hash(key: int, buckets: int) -> int: ...

stubs/txredisapi.pyi

@@ -33,7 +33,9 @@ class RedisProtocol(protocol.Protocol):
         only_if_not_exists: bool = False,
         only_if_exists: bool = False,
     ) -> "Deferred[None]": ...
+    def mset(self, values: dict[str, Any]) -> "Deferred[Any]": ...
     def get(self, key: str) -> "Deferred[Any]": ...
+    def mget(self, keys: list[str]) -> "Deferred[Any]": ...
 
 class SubscriberProtocol(RedisProtocol):
     def __init__(self, *args: object, **kwargs: object): ...

synapse/config/redis.py

@@ -35,6 +35,12 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         self.redis_port = redis_config.get("port", 6379)
         self.redis_password = redis_config.get("password")
 
+        cache_shard_config = redis_config.get("cache_shards")
+        if cache_shard_config:
+            self.cache_shard_hosts = cache_shard_config.get("hosts", [])
+            self.cache_shard_expire = cache_shard_config.get("expire_caches", False)
+            self.cache_shard_ttl = cache_shard_config.get("cache_entry_ttl", False)
+
     def generate_config_section(self, **kwargs: Any) -> str:
         return """\
         # Configuration for Redis when using workers. This *must* be enabled when
@@ -54,4 +60,16 @@ def generate_config_section(self, **kwargs: Any) -> str:
           # Optional password if configured on the Redis instance
           #
           #password: <secret_password>
+
+          # Optional one or more Redis hosts to use for long term shared caches.
+          # Should be configured to automatically expire records when out of
+          # memory, and not be the same instance as used for replication.
+          #
+          #cache_shards:
+          #  enabled: false
+          #  expire_caches: false
+          #  cache_entry_ttl: 30m
+          #  hosts:
+          #    - host: localhost
+          #      port: 6379
         """