Add metrics tracking for eviction to ResponseCache (#16028)

Track whether the ResponseCache is evicting due to invalidation
or due to time.

changelog.d/16028.misc

@@ -0,0 +1 @@
+Collect additional metrics from `ResponseCache` for eviction.

synapse/util/caches/response_cache.py

@@ -36,7 +36,7 @@
 )
 from synapse.util import Clock
 from synapse.util.async_helpers import AbstractObservableDeferred, ObservableDeferred
-from synapse.util.caches import register_cache
+from synapse.util.caches import EvictionReason, register_cache
 
 logger = logging.getLogger(__name__)
 
@@ -167,7 +167,7 @@ def on_complete(r: RV) -> RV:
             # the should_cache bit, we leave it in the cache for now and schedule
             # its removal later.
             if self.timeout_sec and context.should_cache:
-                self.clock.call_later(self.timeout_sec, self.unset, key)
+                self.clock.call_later(self.timeout_sec, self._entry_timeout, key)
             else:
                 # otherwise, remove the result immediately.
                 self.unset(key)
@@ -185,6 +185,12 @@ def unset(self, key: KV) -> None:
         Args:
             key: key used to remove the cached value
         """
+        self._metrics.inc_evictions(EvictionReason.invalidation)
+        self._result_cache.pop(key, None)
+
+    def _entry_timeout(self, key: KV) -> None:
+        """For the call_later to remove from the cache"""
+        self._metrics.inc_evictions(EvictionReason.time)
         self._result_cache.pop(key, None)
 
     async def wrap(