Add support for Valkey and default to using it instead of KeyDB

Hopefully fixes #3544

CHANGELOG.md

@@ -1,3 +1,45 @@
+# 2024-11-23
+
+## (Backward Compatibility Break) The playbook now defaults to Valkey, instead of KeyDB
+
+**TLDR**: if the playbook installed KeyDB (or Redis) as a dependency for you before, it will now replace it with [Valkey](https://valkey.io/) (a drop-in alternative). We [previously switched from Redis to KeyDB](#backward-compatibility-break-the-playbook-now-defaults-to-keydb-instead-of-redis), but Valkey is a better alternative, so we're switching again.
+
+The playbook used to install Redis or KeyDB if services have a need for a Redis-compatible implementation ([enabling worker support for Synapse](docs/configuring-playbook-synapse.md#load-balancing-with-workers), [enabling Hookshot encryption](docs/configuring-playbook-bridge-hookshot.md#end-to-bridge-encryption), etc.).
+
+Earlier this year, we switched from Redis to KeyDB - see [(Backward Compatibility Break) The playbook now defaults to KeyDB, instead of Redis](#backward-compatibility-break-the-playbook-now-defaults-to-keydb-instead-of-redis).
+
+Because Valkey seems to be a better successor to Redis (than KeyDB) and likely doesn't suffer from [issues like this one](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3544), we now replace KeyDB with Valkey.
+
+Valkey (like KeyDB and Redis in the past) is an implicitly enabled dependency - you don't need custom configuration in `vars.yml` to enable it.
+
+Next time your run the playbook (via the `setup-all` tag), **KeyDB will be automatically uninstalled and replaced with Valkey**. Some Synapse downtime may occur while the switch happens.
+
+Users on `arm32` should be aware that there's **neither a prebuilt `arm32` container image for Valkey**, nor the Valkey role supports self-building yet. Users on this architecture likely don't run Synapse with workers, etc., so they're likely in no need of Valkey (or Redis/KeyDB). If Redis is necessary in an `arm32` deployment, disabling Valkey and making the playbook fall back to Redis is possible (see below).
+
+**The playbook still supports Redis** and you can keep using Redis (for now) if you'd like, by adding this additional configuration to your `vars.yml` file:
+
+```yml
+# Explicitly disable both Valkey and KeyDB.
+#
+# Redis will be auto-enabled if necessary,
+# because there's no other Redis-compatible implementation being enabled.
+valkey_enabled: false
+keydb_enabled: false
+```
+
+**The playbook still supports KeyDB** and you can keep using KeyDB (for now) if you'd like, by adding this additional configuration to your `vars.yml` file:
+
+```yml
+# Explicitly disable Valkey  enable KeyDB.
+#
+# Redis will not be auto-enabled beandcause a Redis-compatible implementation (KeyDB) is enabled.
+valkey_enabled: false
+keydb_enabled: true
+```
+
+At some point in time in the future, we'll remove both KeyDB and Redis from the playbook, so we recommend that you migrate to Valkey earlier anyway.
+
+
 # 2024-11-14
 
 ## HTTP-compression support for Traefik-based setups

group_vars/matrix_servers

@@ -436,6 +436,8 @@ devture_systemd_service_manager_services_list_auto: |
     +
     ([{'name': (keydb_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'keydb']}] if keydb_enabled else [])
     +
+    ([{'name': (valkey_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'valkey']}] if valkey_enabled else [])
+    +
     ([{'name': 'matrix-pantalaimon.service', 'priority': 4000, 'groups': ['matrix', 'pantalaimon']}] if matrix_pantalaimon_enabled else [])
     +
     ([{'name': 'matrix-registration.service', 'priority': 4000, 'groups': ['matrix', 'registration', 'matrix-registration']}] if matrix_registration_enabled else [])
@@ -2203,12 +2205,14 @@ matrix_hookshot_systemd_wanted_services_list: |
     ([(redis_identifier + '.service')] if redis_enabled and matrix_hookshot_cache_redis_host == redis_identifier else [])
     +
     ([(keydb_identifier + '.service')] if keydb_enabled and matrix_hookshot_cache_redis_host == keydb_identifier else [])
+    +
+    ([(valkey_identifier + '.service')] if valkey_enabled and matrix_hookshot_cache_redis_host == valkey_identifier else [])
   }}
 
 # Hookshot's experimental encryption feature (and possibly others) may benefit from Redis, if available.
 # We only connect to Redis if encryption is enabled (not for everyone who has Redis enabled),
 # because connectivity is still potentially troublesome and is to be investigated.
-matrix_hookshot_cache_redis_host: "{{ redis_identifier if redis_enabled and matrix_hookshot_experimental_encryption_enabled else (keydb_identifier if keydb_enabled and matrix_hookshot_experimental_encryption_enabled else '') }}"
+matrix_hookshot_cache_redis_host: "{{ valkey_identifier if valkey_enabled else (redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '')) }}"
 
 matrix_hookshot_container_network: "{{ matrix_addons_container_network }}"
 
@@ -2221,6 +2225,8 @@ matrix_hookshot_container_additional_networks_auto: |
       +
       ([keydb_container_network] if keydb_enabled and matrix_hookshot_cache_redis_host == keydb_identifier else [])
       +
+      ([valkey_container_network] if valkey_enabled and matrix_hookshot_cache_redis_host == valkey_identifier else [])
+      +
       ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_playbook_reverse_proxyable_services_additional_network and matrix_hookshot_container_labels_traefik_enabled else [])
     ) | unique
   }}
@@ -4393,11 +4399,11 @@ ntfy_visitor_request_limit_exempt_hosts_hostnames_auto: |
 
 ######################################################################
 #
-# etke/redis
+# redis
 #
 ######################################################################
 
-redis_enabled: "{{ not keydb_enabled and (matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled)) }}"
+redis_enabled: "{{ not (keydb_enabled or valkey_enabled) and (matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled)) }}"
 
 redis_identifier: matrix-redis
 
@@ -4408,7 +4414,7 @@ redis_base_path: "{{ matrix_base_data_path }}/redis"
 
 ######################################################################
 #
-# /etke/redis
+# /redis
 #
 ######################################################################
 
@@ -4418,7 +4424,7 @@ redis_base_path: "{{ matrix_base_data_path }}/redis"
 #
 ######################################################################
 
-keydb_enabled: "{{ matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled) }}"
+keydb_enabled: false
 
 keydb_identifier: matrix-keydb
 
@@ -4442,6 +4448,31 @@ keydb_arch: |-
 #
 ######################################################################
 
+
+######################################################################
+#
+# valkey
+#
+######################################################################
+
+valkey_enabled: "{{ matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled) }}"
+
+valkey_identifier: matrix-valkey
+
+valkey_uid: "{{ matrix_user_uid }}"
+valkey_gid: "{{ matrix_user_gid }}"
+
+valkey_base_path: "{{ matrix_base_data_path }}/valkey"
+
+valkey_arch: "{{ matrix_architecture }}"
+
+######################################################################
+#
+# valkey
+#
+######################################################################
+
+
 ######################################################################
 #
 # matrix-client-element
@@ -4664,6 +4695,8 @@ matrix_synapse_container_additional_networks_auto: |
       +
       ([keydb_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == keydb_identifier else [])
       +
+      ([valkey_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == valkey_identifier else [])
+      +
       ([exim_relay_container_network] if (exim_relay_enabled and matrix_synapse_email_enabled and matrix_synapse_email_smtp_host == exim_relay_identifier and matrix_synapse_container_network != exim_relay_container_network) else [])
       +
       ([matrix_ma1sd_container_network] if (matrix_ma1sd_enabled and matrix_synapse_account_threepid_delegates_msisdn == matrix_synapse_account_threepid_delegates_msisdn_mas1sd_url and matrix_synapse_container_network != matrix_ma1sd_container_network) else [])
@@ -4751,6 +4784,8 @@ matrix_synapse_systemd_required_services_list_auto: |
     +
     ([keydb_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == keydb_identifier else [])
     +
+    ([valkey_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == valkey_identifier else [])
+    +
     (['matrix-goofys.service'] if matrix_s3_media_store_enabled else [])
     +
     (['matrix-authentication-service.service'] if (matrix_authentication_service_enabled and matrix_synapse_experimental_features_msc3861_enabled) else [])
@@ -4764,9 +4799,9 @@ matrix_synapse_systemd_wanted_services_list_auto: |
   }}
 
 # Synapse workers (used for parallel load-scaling) need Redis for IPC.
-matrix_synapse_redis_enabled: "{{ redis_enabled or keydb_enabled }}"
-matrix_synapse_redis_host: "{{ redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '') }}"
-matrix_synapse_redis_password: "{{ redis_connection_password if redis_enabled else (keydb_connection_password if keydb_enabled else '') }}"
+matrix_synapse_redis_enabled: "{{ redis_enabled or keydb_enabled or valkey_enabled }}"
+matrix_synapse_redis_host: "{{ valkey_identifier if valkey_enabled else (redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '')) }}"
+matrix_synapse_redis_password: "{{ valkey_connection_password if valkey_enabled else (redis_connection_password if redis_enabled else (keydb_connection_password if keydb_enabled else '')) }}"
 
 matrix_synapse_container_extra_arguments_auto: "{{ matrix_homeserver_container_extra_arguments_auto }}"
 matrix_synapse_app_service_config_files_auto: "{{ matrix_homeserver_app_service_config_files_auto }}"

requirements.yml

@@ -75,3 +75,6 @@
 - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-traefik-certs-dumper.git
   version: v2.8.3-5
   name: traefik_certs_dumper
+- src: git+https://github.com/mother-of-all-self-hosting/ansible-role-valkey.git
+  version: v8.0.1-0
+  name: valkey

setup.yml

@@ -49,6 +49,8 @@
 
     - galaxy/redis
     - galaxy/keydb
+    - galaxy/valkey
+
     - role: custom/matrix-authentication-service
     - custom/matrix-corporal
     - custom/matrix-appservice-draupnir-for-all