From 7f9c973c6540f45b6aa99b17a38ea261b3277a70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 10:38:36 +0000 Subject: [PATCH 1/6] Add index to `local_group_updates.stream_id` This should speed up startup times and generally increase performance of groups. --- synapse/storage/databases/main/group_server.py | 17 ++++++++++++++++- .../main/delta/65/03_local_group_updates.sql | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/main/delta/65/03_local_group_updates.sql diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py index e70d3649ff5a..58f41819246c 100644 --- a/synapse/storage/databases/main/group_server.py +++ b/synapse/storage/databases/main/group_server.py @@ -13,15 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from typing_extensions import TypedDict from synapse.api.errors import SynapseError from synapse.storage._base import SQLBaseStore, db_to_json +from synapse.storage.database import DatabasePool +from synapse.storage.types import Connection from synapse.types import JsonDict from synapse.util import json_encoder +if TYPE_CHECKING: + from synapse.server import HomeServer + # The category ID for the "default" category. We don't store as null in the # database to avoid the fun of null != null _DEFAULT_CATEGORY_ID = "" @@ -35,6 +40,16 @@ class _RoomInGroup(TypedDict): class GroupServerWorkerStore(SQLBaseStore): + def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"): + database.updates.register_background_index_update( + update_name="local_group_updates_index", + index_name="local_group_updates_stream_id_index", + table="local_group_updates", + columns=("stream_id"), + unique=True, + ) + super().__init__(database, db_conn, hs) + async def get_group(self, group_id: str) -> Optional[Dict[str, Any]]: return await self.db_pool.simple_select_one( table="groups", diff --git a/synapse/storage/schema/main/delta/65/03_local_group_updates.sql b/synapse/storage/schema/main/delta/65/03_local_group_updates.sql new file mode 100644 index 000000000000..3000a6fcc6a0 --- /dev/null +++ b/synapse/storage/schema/main/delta/65/03_local_group_updates.sql @@ -0,0 +1,18 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Check old events for thread relations. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (6503, 'local_group_updates_index', '{}'); From 15e1b39765f29383fc0fa6028b2bacd5d83e2dbc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 10:41:06 +0000 Subject: [PATCH 2/6] Newsfile --- changelog.d/11231.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11231.misc diff --git a/changelog.d/11231.misc b/changelog.d/11231.misc new file mode 100644 index 000000000000..96c29e3434f0 --- /dev/null +++ b/changelog.d/11231.misc @@ -0,0 +1 @@ +Minor speed up to start up times and getting updates for groups, by adding missing index to `local_group_updates.stream_id`. From 914103bc6a0fc09f25e974f77369b473cc4c0456 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 11:10:02 +0000 Subject: [PATCH 3/6] Update changelog.d/11231.misc Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/11231.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/11231.misc b/changelog.d/11231.misc index 96c29e3434f0..c7fca7071eda 100644 --- a/changelog.d/11231.misc +++ b/changelog.d/11231.misc @@ -1 +1 @@ -Minor speed up to start up times and getting updates for groups, by adding missing index to `local_group_updates.stream_id`. +Minor speed up to start up times and getting updates for groups by adding missing index to `local_group_updates.stream_id`. From c6ee71a48c2b90715f8678e8596bff2913befa9b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 11:10:07 +0000 Subject: [PATCH 4/6] Review comments --- synapse/storage/databases/main/group_server.py | 2 +- synapse/storage/schema/main/delta/65/03_local_group_updates.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py index 58f41819246c..bb621df0ddb6 100644 --- a/synapse/storage/databases/main/group_server.py +++ b/synapse/storage/databases/main/group_server.py @@ -45,7 +45,7 @@ def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer" update_name="local_group_updates_index", index_name="local_group_updates_stream_id_index", table="local_group_updates", - columns=("stream_id"), + columns=("stream_id",), unique=True, ) super().__init__(database, db_conn, hs) diff --git a/synapse/storage/schema/main/delta/65/03_local_group_updates.sql b/synapse/storage/schema/main/delta/65/03_local_group_updates.sql index 3000a6fcc6a0..a32aa79e9768 100644 --- a/synapse/storage/schema/main/delta/65/03_local_group_updates.sql +++ b/synapse/storage/schema/main/delta/65/03_local_group_updates.sql @@ -13,6 +13,6 @@ * limitations under the License. */ --- Check old events for thread relations. +-- Check index on `local_group_updates.stream_id`. INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (6503, 'local_group_updates_index', '{}'); From e57a30bdd1818ff43abdf88420b3a31be081c38f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 12:03:58 +0000 Subject: [PATCH 5/6] Fix portdb --- scripts/synapse_port_db | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 349866eb9a82..640ff15277db 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -43,6 +43,7 @@ from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackground from synapse.storage.databases.main.events_bg_updates import ( EventsBackgroundUpdatesStore, ) +from synapse.storage.databases.main.group_server import GroupServerWorkerStore from synapse.storage.databases.main.media_repository import ( MediaRepositoryBackgroundUpdateStore, ) @@ -181,6 +182,7 @@ class Store( StatsStore, PusherWorkerStore, PresenceBackgroundUpdateStore, + GroupServerWorkerStore, ): def execute(self, f, *args, **kwargs): return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs) From a15e0106ed1777c3d8987764d03a67813f0dbee8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Nov 2021 15:04:34 +0000 Subject: [PATCH 6/6] Move to 04 --- .../{03_local_group_updates.sql => 04_local_group_updates.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/main/delta/65/{03_local_group_updates.sql => 04_local_group_updates.sql} (94%) diff --git a/synapse/storage/schema/main/delta/65/03_local_group_updates.sql b/synapse/storage/schema/main/delta/65/04_local_group_updates.sql similarity index 94% rename from synapse/storage/schema/main/delta/65/03_local_group_updates.sql rename to synapse/storage/schema/main/delta/65/04_local_group_updates.sql index a32aa79e9768..a178abfe12d6 100644 --- a/synapse/storage/schema/main/delta/65/03_local_group_updates.sql +++ b/synapse/storage/schema/main/delta/65/04_local_group_updates.sql @@ -15,4 +15,4 @@ -- Check index on `local_group_updates.stream_id`. INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (6503, 'local_group_updates_index', '{}'); + (6504, 'local_group_updates_index', '{}');