From ac3767d7649f3204cb960624ac978a42eaffbb3f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 23 Jan 2023 15:48:08 +0000 Subject: [PATCH 1/5] Request partial joins by default This is a little sloppy, but we are trying to gain confidence in faster joins in the upcoming RC. Admins can still opt out by adding the following to their Synapse config: ```yaml experimental: faster_joins: false ``` We may revert this change before the release proper, depending on how testing in the wild goes. --- synapse/config/experimental.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 89586db76313..2590c88cdeb1 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -84,7 +84,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # experimental support for faster joins over federation # (MSC2775, MSC3706, MSC3895) # requires a target server that can provide a partial join response (MSC3706) - self.faster_joins_enabled: bool = experimental.get("faster_joins", False) + self.faster_joins_enabled: bool = experimental.get("faster_joins", True) # MSC3720 (Account status endpoint) self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) From adc1e3c5cd21454edbb533cf03223fd7a621c58b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 23 Jan 2023 16:03:30 +0000 Subject: [PATCH 2/5] Changelog --- changelog.d/14905.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14905.feature diff --git a/changelog.d/14905.feature b/changelog.d/14905.feature new file mode 100644 index 000000000000..f13a4af9815a --- /dev/null +++ b/changelog.d/14905.feature @@ -0,0 +1 @@ +Faster joins: request partial joins by default. Admins can opt-out of this for the time being---see the upgrade notes. From ef873c05fdc1ed39a8f5b266deb096f3ee5b92a5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 24 Jan 2023 13:10:28 +0000 Subject: [PATCH 3/5] Try to fix the backfill test failures --- synapse/storage/databases/main/stream.py | 31 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 63d835053019..de85270b7da5 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -944,12 +944,33 @@ async def get_current_topological_token(self, room_id: str, stream_key: int) -> room_id stream_key """ - sql = ( - "SELECT coalesce(MIN(topological_ordering), 0) FROM events" - " WHERE room_id = ? AND stream_ordering >= ?" - ) + # This query used to be + # SELECT COALESCE(MIN(topological_ordering), 0) FROM events + # WHERE room_id = ? and events.stream_ordering >= {stream_key} + # which returns 0 if the stream_key is newer than any event in + # the room. That's not wrong, but it seems to interact oddly with backfill, + # requiring a second call to /messages to actually backfill from a remote + # homeserver. + # + # Instead, rollback the stream ordering to that after the most recent event in + # this room. + sql = """ + WITH fallback(max_stream_ordering) AS ( + SELECT MAX(stream_ordering) + FROM events + WHERE room_id = ? + ) + SELECT COALESCE(MIN(topological_ordering), 0) FROM events + WHERE + room_id = ? + AND events.stream_ordering >= MIN( + ?, + (SELECT max_stream_ordering FROM fallback) + ) + """ + row = await self.db_pool.execute( - "get_current_topological_token", None, sql, room_id, stream_key + "get_current_topological_token", None, sql, room_id, room_id, stream_key ) return row[0][0] if row else 0 From 5eef1fa3c08fb88b6f6b127ddd206197caa81f0b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 24 Jan 2023 13:16:44 +0000 Subject: [PATCH 4/5] Upgrade notes --- docs/upgrade.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 0d486a3c8200..6316db563b1a 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -90,6 +90,19 @@ process, for example: # Upgrading to v1.76.0 +## Faster joins are enabled by default + +When joining a room for the first time, Synapse 1.76.0rc1 will request a partial join from the other server by default. Previously, server admins had to opt-in to this using an experimental config flag. + +Server admins can opt out of this feature for the time being by setting + +```yaml +experimental: + faster_joins: false +``` + +in their server config. + ## Changes to the account data replication streams Synapse has changed the format of the account data and devices replication From a8f4795a8445ac838be26303606b55b078272a5f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 24 Jan 2023 14:02:35 +0000 Subject: [PATCH 5/5] Postgres compat? --- synapse/storage/databases/main/stream.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index de85270b7da5..d28fc65df9bb 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -67,7 +67,7 @@ make_in_list_sql_clause, ) from synapse.storage.databases.main.events_worker import EventsWorkerStore -from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import PersistedEventPosition, RoomStreamToken from synapse.util.caches.descriptors import cached @@ -944,6 +944,13 @@ async def get_current_topological_token(self, room_id: str, stream_key: int) -> room_id stream_key """ + if isinstance(self.database_engine, PostgresEngine): + min_function = "LEAST" + elif isinstance(self.database_engine, Sqlite3Engine): + min_function = "MIN" + else: + raise RuntimeError(f"Unknown database engine {self.database_engine}") + # This query used to be # SELECT COALESCE(MIN(topological_ordering), 0) FROM events # WHERE room_id = ? and events.stream_ordering >= {stream_key} @@ -954,7 +961,7 @@ async def get_current_topological_token(self, room_id: str, stream_key: int) -> # # Instead, rollback the stream ordering to that after the most recent event in # this room. - sql = """ + sql = f""" WITH fallback(max_stream_ordering) AS ( SELECT MAX(stream_ordering) FROM events @@ -963,7 +970,7 @@ async def get_current_topological_token(self, room_id: str, stream_key: int) -> SELECT COALESCE(MIN(topological_ordering), 0) FROM events WHERE room_id = ? - AND events.stream_ordering >= MIN( + AND events.stream_ordering >= {min_function}( ?, (SELECT max_stream_ordering FROM fallback) )