Allow configuration of Synapse's cache without using synctl or environment variables

changelog.d/6391.feature

@@ -0,0 +1 @@
+Synapse's cache factor can now be configured in `homeserver.yaml` by the `caches.global_factor` setting. Additionally, `caches.per_cache_factors` controls the cache factors for individual caches.

synapse/api/auth.py

@@ -41,7 +41,7 @@
 )
 from synapse.config.server import is_threepid_reserved
 from synapse.types import UserID
-from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
+from synapse.util.caches import register_cache
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.metrics import Measure
 
@@ -77,7 +77,7 @@ def __init__(self, hs):
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
 
-        self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
+        self.token_cache = LruCache(10000)
         register_cache("cache", "token_cache", self.token_cache)
 
         self._account_validity = hs.config.account_validity

synapse/app/homeserver.py

@@ -71,7 +71,6 @@
 from synapse.storage import DataStore, are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
 from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
-from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
@@ -516,7 +515,7 @@ def phone_stats_home(hs, stats, stats_process=_stats_process):
 
     daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
     stats["daily_sent_messages"] = daily_sent_messages
-    stats["cache_factor"] = CACHE_SIZE_FACTOR
+    stats["cache_factor"] = hs.config.caches.global_factor
     stats["event_cache_size"] = hs.config.event_cache_size
 
     #

synapse/config/cache.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from collections import defaultdict
+from typing import DefaultDict
+
+from twisted.logger import Logger
+
+from ._base import Config, ConfigError
+
+log = Logger()
+
+_CACHES = {}
+DEFAULT_CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
+
+
+_DEFAULT_CONFIG = """\
+# Cache configuration
+#
+# 'global_factor' controls the global cache factor. This overrides the
+# "SYNAPSE_CACHE_FACTOR" environment variable.
+#
+# 'per_cache_factors' is a dictionary of cache name to cache factor for that
+# individual cache.
+#
+#caches:
+#  global_factor: 0.5
+#  per_cache_factors:
+#    get_users_who_share_room_with_user: 2
+#
+"""
+
+
+def add_resizable_cache(cache_name, cache_resize_callback):
+    _CACHES[cache_name.lower()] = cache_resize_callback
+
+
+class CacheConfig(Config):
+    section = "caches"
+
+    def read_config(self, config, **kwargs):
+        global DEFAULT_CACHE_SIZE_FACTOR
+
+        cache_config = config.get("caches", {})
+
+        self.global_factor = cache_config.get(
+            "global_factor", DEFAULT_CACHE_SIZE_FACTOR
+        )
+        if not isinstance(self.global_factor, (int, float)):
+            raise ConfigError("caches.global_factor must be a number.")
+
+        # Set the global one so that it's reflected in new caches
+        DEFAULT_CACHE_SIZE_FACTOR = self.global_factor
+
+        individual_factors = cache_config.get("per_cache_factors", {}) or {}
+        if not isinstance(individual_factors, dict):
+            raise ConfigError("caches.per_cache_factors must be a dictionary")
+
+        self.cache_factors = defaultdict(
+            lambda: self.global_factor
+        )  # type: DefaultDict[str, float]
+
+        for cache, factor in individual_factors.items():
+            if not isinstance(factor, (int, float)):
+                raise ConfigError(
+                    "caches.per_cache_factors.%s must be a number" % (cache.lower(),)
+                )
+            self.cache_factors[cache.lower()] = factor
+
+    def resize_caches(self):
+        for cache_name, cache_resize_callback in _CACHES.items():
+            cache_factor = self.cache_factors[cache_name]
+            log.debug(
+                "Setting cache factor for {cache_name} to {new_cache_factor}",
+                cache_name=cache_name,
+                new_cache_factor=cache_factor,
+            )
+            changed = cache_resize_callback(cache_factor)
+            if changed:
+                log.info(
+                    "Cache factor for {cache_name} set to {new_cache_factor}",
+                    cache_name=cache_name,
+                    new_cache_factor=cache_factor,
+                )
+
+    def get_factor_for(self, cache_name):
+        return self.cache_factors[cache_name.lower()]

synapse/config/homeserver.py

@@ -17,6 +17,7 @@
 from ._base import RootConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
+from .cache import CacheConfig
 from .captcha import CaptchaConfig
 from .cas import CasConfig
 from .consent_config import ConsentConfig
@@ -80,4 +81,5 @@ class HomeServerConfig(RootConfig):
         RoomDirectoryConfig,
         ThirdPartyRulesConfig,
         TracerConfig,
+        CacheConfig,
     ]

synapse/http/client.py

@@ -49,7 +49,6 @@
 from synapse.logging.context import make_deferred_yieldable
 from synapse.logging.opentracing import set_tag, start_active_span, tags
 from synapse.util.async_helpers import timeout_deferred
-from synapse.util.caches import CACHE_SIZE_FACTOR
 
 logger = logging.getLogger(__name__)
 
@@ -241,7 +240,8 @@ def __getattr__(_self, attr):
         # tends to do so in batches, so we need to allow the pool to keep
         # lots of idle connections around.
         pool = HTTPConnectionPool(self.reactor)
-        pool.maxPersistentPerHost = max((100 * CACHE_SIZE_FACTOR, 5))
+        # XXX: Why does this use the cache factor????
+        pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5))
         pool.cachedConnectionTimeout = 2 * 60
 
         # The default context factory in Twisted 14.0.0 (which we require) is

synapse/metrics/_exposition.py

@@ -33,6 +33,8 @@
 
 from twisted.web.resource import Resource
 
+from synapse.util import caches
+
 try:
     from prometheus_client.samples import Sample
 except ImportError:
@@ -103,13 +105,15 @@ def nameify_sample(sample):
 
 
 def generate_latest(registry, emit_help=False):
-    output = []
 
-    for metric in registry.collect():
+    # Trigger the cache metrics to be rescraped, which updates the common
+    # metrics but do not produce metrics themselves
+    for collector in caches.collectors_by_name.values():
+        collector.collect()
 
-        if metric.name.startswith("__unused"):
-            continue
+    output = []
 
+    for metric in registry.collect():
         if not metric.samples:
             # No samples, don't bother.
             continue

synapse/push/bulk_push_rule_evaluator.py

@@ -51,6 +51,7 @@
     "cache",
     "push_rules_delta_state_cache_metric",
     cache=[],  # Meaningless size, as this isn't a cache that stores values
+    resizable=False,
 )
 
 
@@ -67,7 +68,8 @@ def __init__(self, hs):
         self.room_push_rule_cache_metrics = register_cache(
             "cache",
             "room_push_rule_cache",
-            cache=[],  # Meaningless size, as this isn't a cache that stores values
+            cache=[],  # Meaningless size, as this isn't a cache that stores values,
+            resizable=False,
         )
 
     @defer.inlineCallbacks

synapse/push/push_rule_evaluator.py

@@ -20,7 +20,7 @@
 from six import string_types
 
 from synapse.types import UserID
-from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
+from synapse.util.caches import register_cache
 from synapse.util.caches.lrucache import LruCache
 
 logger = logging.getLogger(__name__)
@@ -149,7 +149,7 @@ def _get_value(self, dotted_key):
 
 
 # Caches (glob, word_boundary) -> regex for push. See _glob_matches
-regex_cache = LruCache(50000 * CACHE_SIZE_FACTOR)
+regex_cache = LruCache(50000)
 register_cache("cache", "regex_push_cache", regex_cache)
 
 

synapse/replication/slave/storage/client_ips.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 from synapse.storage.data_stores.main.client_ips import LAST_SEEN_GRANULARITY
-from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.caches.descriptors import Cache
 
 from ._base import BaseSlavedStore
@@ -25,7 +24,7 @@ def __init__(self, db_conn, hs):
         super(SlavedClientIpStore, self).__init__(db_conn, hs)
 
         self.client_ip_last_seen = Cache(
-            name="client_ip_last_seen", keylen=4, max_entries=50000 * CACHE_SIZE_FACTOR
+            name="client_ip_last_seen", keylen=4, max_entries=50000
         )
 
     def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id):

synapse/server.py

@@ -242,6 +242,7 @@ def setup(self):
             self.datastores = DataStores(datastore, conn, self)
             conn.commit()
         self.start_time = int(self.get_clock().time())
+        self.get_clock().looping_call(self.config.caches.resize_caches, 10000)
         logger.info("Finished setting up.")
 
     def setup_master(self):

synapse/state/__init__.py

@@ -33,7 +33,6 @@
 from synapse.logging.utils import log_function
 from synapse.state import v1, v2
 from synapse.util.async_helpers import Linearizer
-from synapse.util.caches import get_cache_factor_for
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.metrics import Measure, measure_func
 
@@ -51,7 +50,6 @@
 KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key"))
 
 
-SIZE_OF_CACHE = 100000 * get_cache_factor_for("state_cache")
 EVICTION_TIMEOUT_SECONDS = 60 * 60
 
 
@@ -441,10 +439,11 @@ def __init__(self, hs):
         self._state_cache = None
         self.resolve_linearizer = Linearizer(name="state_resolve_lock")
 
+        cache_size = 100000 * hs.config.caches.get_factor_for("state_cache")
         self._state_cache = ExpiringCache(
             cache_name="state_cache",
             clock=self.clock,
-            max_len=SIZE_OF_CACHE,
+            max_len=cache_size,
             expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000,
             iterable=True,
             reset_expiry_on_get=True,

synapse/storage/data_stores/main/client_ips.py

@@ -22,7 +22,6 @@
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.storage import background_updates
 from synapse.storage._base import Cache
-from synapse.util.caches import CACHE_SIZE_FACTOR
 
 logger = logging.getLogger(__name__)
 
@@ -366,7 +365,7 @@ class ClientIpStore(ClientIpBackgroundUpdateStore):
     def __init__(self, db_conn, hs):
 
         self.client_ip_last_seen = Cache(
-            name="client_ip_last_seen", keylen=4, max_entries=50000 * CACHE_SIZE_FACTOR
+            name="client_ip_last_seen", keylen=4, max_entries=50000
         )
 
         super(ClientIpStore, self).__init__(db_conn, hs)

synapse/storage/data_stores/main/state.py

@@ -31,7 +31,7 @@
 from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.state import StateFilter
-from synapse.util.caches import get_cache_factor_for, intern_string
+from synapse.util.caches import intern_string
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.dictionary_cache import DictionaryCache
 from synapse.util.stringutils import to_ascii
@@ -246,14 +246,10 @@ def __init__(self, db_conn, hs):
         # We size the non-members cache to be smaller than the members cache as the
         # vast majority of state in Matrix (today) is member events.
 
-        self._state_group_cache = DictionaryCache(
-            "*stateGroupCache*",
-            # TODO: this hasn't been tuned yet
-            50000 * get_cache_factor_for("stateGroupCache"),
-        )
+        # TODO: this hasn't been tuned yet
+        self._state_group_cache = DictionaryCache("*stateGroupCache*", 50000)
         self._state_group_members_cache = DictionaryCache(
-            "*stateGroupMembersCache*",
-            500000 * get_cache_factor_for("stateGroupMembersCache"),
+            "*stateGroupMembersCache*", 500000
         )
 
     @defer.inlineCallbacks