From 432dcfbbedf4da07a055a1bf302829108a25507f Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 09:50:34 +0200 Subject: [PATCH 1/9] Update SQL statements in docs for Synapse Admins --- .../administration/useful_sql_for_admins.md | 220 +++++++++++++----- 1 file changed, 166 insertions(+), 54 deletions(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index d4aada3272d0..8622a7199ac0 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -1,7 +1,10 @@ ## Some useful SQL queries for Synapse Admins ## Size of full matrix db -`SELECT pg_size_pretty( pg_database_size( 'matrix' ) );` +```sql +SELECT + pg_size_pretty( pg_database_size( 'matrix' ) ); +``` ### Result example: ``` pg_size_pretty @@ -11,37 +14,64 @@ pg_size_pretty ``` ## Show top 20 larger rooms by state events count ```sql -SELECT r.name, s.room_id, s.current_state_events - FROM room_stats_current s - LEFT JOIN room_stats_state r USING (room_id) - ORDER BY current_state_events DESC - LIMIT 20; +SELECT + r.name, + s.room_id, + s.current_state_events +FROM + room_stats_current s + LEFT JOIN + room_stats_state r USING (room_id) +ORDER BY + current_state_events DESC LIMIT 20; ``` and by state_group_events count: ```sql -SELECT rss.name, s.room_id, count(s.room_id) FROM state_groups_state s -LEFT JOIN room_stats_state rss USING (room_id) -GROUP BY s.room_id, rss.name -ORDER BY count(s.room_id) DESC -LIMIT 20; +SELECT + rss.name, + s.room_id, + COUNT(s.room_id) +FROM + state_groups_state s + LEFT JOIN + room_stats_state rss USING (room_id) +GROUP BY + s.room_id, + rss.name +ORDER BY + COUNT(s.room_id) DESC LIMIT 20; ``` plus same, but with join removed for performance reasons: ```sql -SELECT s.room_id, count(s.room_id) FROM state_groups_state s -GROUP BY s.room_id -ORDER BY count(s.room_id) DESC -LIMIT 20; +SELECT + s.room_id, + COUNT(s.room_id) +FROM + state_groups_state s +GROUP BY + s.room_id +ORDER BY + COUNT(s.room_id) DESC LIMIT 20; ``` ## Show top 20 larger tables by row count ```sql -SELECT relname, n_live_tup as rows - FROM pg_stat_user_tables - ORDER BY n_live_tup DESC - LIMIT 20; +SELECT + relname, + n_live_tup AS "rows" +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC LIMIT 20; +``` +This query is quick, but may be very approximate, for exact number of rows use: +```sql +SELECT + COUNT(*) +FROM + ; ``` -This query is quick, but may be very approximate, for exact number of rows use `SELECT COUNT(*) FROM `. ### Result example: ``` state_groups_state - 161687170 @@ -68,43 +98,84 @@ user_directory_search - 316433 ## Show top 20 rooms by new events count in last 1 day: ```sql -SELECT e.room_id, r.name, COUNT(e.event_id) cnt FROM events e -LEFT JOIN room_stats_state r USING (room_id) -WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 GROUP BY e.room_id, r.name ORDER BY cnt DESC LIMIT 20; +SELECT + e.room_id, + r.name, + COUNT(e.event_id) cnt +FROM + events e + LEFT JOIN + room_stats_state r USING (room_id) +WHERE + e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 +GROUP BY + e.room_id, + r.name +ORDER BY + cnt DESC LIMIT 20; ``` ## Show top 20 users on homeserver by sent events (messages) at last month: ```sql -SELECT user_id, SUM(total_events) - FROM user_stats_historical - WHERE TO_TIMESTAMP(end_ts/1000) AT TIME ZONE 'UTC' > date_trunc('day', now() - interval '1 month') - GROUP BY user_id - ORDER BY SUM(total_events) DESC - LIMIT 20; +SELECT + COUNT(*), + sender +FROM + events +WHERE + ( + type = 'm.room.encrypted' + OR type = 'm.room.message' + ) + AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 +GROUP BY + sender +ORDER BY + COUNT(*) DESC LIMIT 20; ``` ## Show last 100 messages from needed user, with room names: ```sql -SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json FROM events e - LEFT JOIN event_json j USING (room_id) - LEFT JOIN room_stats_state r USING (room_id) - WHERE sender = '@LOGIN:example.com' - AND e.type = 'm.room.message' - ORDER BY stream_ordering DESC - LIMIT 100; +SELECT + e.room_id, + r.name, + e.event_id, + e.type, + e.content, + j.json +FROM + events e + LEFT JOIN + event_json j USING (room_id) + LEFT JOIN + room_stats_state r USING (room_id) +WHERE + sender = '@LOGIN:example.com' + AND e.type = 'm.room.message' +ORDER BY + stream_ordering DESC LIMIT 100; ``` ## Show top 20 larger tables by storage size ```sql -SELECT nspname || '.' || relname AS "relation", - pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size" - FROM pg_class C - LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) - WHERE nspname NOT IN ('pg_catalog', 'information_schema') - AND C.relkind <> 'i' - AND nspname !~ '^pg_toast' - ORDER BY pg_total_relation_size(C.oid) DESC - LIMIT 20; +SELECT + nspname || '.' || relname AS "relation", + pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" +FROM + pg_class c + LEFT JOIN + pg_namespace n + ON (n.oid = c.relnamespace) +WHERE + nspname NOT IN + ( + 'pg_catalog', + 'information_schema' + ) + AND c.relkind <> 'i' + AND nspname !~ '^pg_toast' +ORDER BY + pg_total_relation_size(c.oid) DESC LIMIT 20; ``` ### Result example: ``` @@ -131,7 +202,33 @@ public.state_group_edges - 122 MB ``` ## Show rooms with names, sorted by events in this rooms -`echo "select event_json.room_id,room_stats_state.name from event_json,room_stats_state where room_stats_state.room_id=event_json.room_id" | psql synapse | sort | uniq -c | sort -n` + +**Sort and order with bash** +```bash +echo "SELECT event_json.room_id, room_stats_state.name FROM event_json, room_stats_state \ +WHERE room_stats_state.room_id = event_json.room_id" | psql -d synapse -h localhost -U synapse_user -t \ +| sort | uniq -c | sort -n +``` +Documentation for `psql` command line parameters: https://www.postgresql.org/docs/current/app-psql.html + +**Sort and order with SQL** +```sql +SELECT + COUNT(*), + event_json.room_id, + room_stats_state.name +FROM + event_json, + room_stats_state +WHERE + room_stats_state.room_id = event_json.room_id +GROUP BY + event_json.room_id, + room_stats_state.name +ORDER BY + COUNT(*) DESC LIMIT 50; +``` + ### Result example: ``` 9459 !FPUfgzXYWTKgIrwKxW:matrix.org | This Week in Matrix @@ -146,11 +243,26 @@ public.state_group_edges - 122 MB ## Lookup room state info by list of room_id ```sql -SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, rsc.joined_members, rsc.local_users_in_room, rss.join_rules -FROM room_stats_state rss -LEFT JOIN room_stats_current rsc USING (room_id) -WHERE room_id IN (WHERE room_id IN ( - '!OGEhHVWSdvArJzumhm:matrix.org', - '!YTvKGNlinIzlkMTVRl:matrix.org' -) -``` \ No newline at end of file +SELECT + rss.room_id, + rss.name, + rss.canonical_alias, + rss.topic, + rss.encryption, + rsc.joined_members, + rsc.local_users_in_room, + rss.join_rules +FROM + room_stats_state rss + LEFT JOIN + room_stats_current rsc USING (room_id) +WHERE + room_id IN + ( + WHERE + room_id IN + ( + '!OGEhHVWSdvArJzumhm:matrix.org', + '!YTvKGNlinIzlkMTVRl:matrix.org' + ); +``` From d09bc04e57943912cb1b6e27a5a987a506882d86 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 09:58:30 +0200 Subject: [PATCH 2/9] changelog --- changelog.d/12536.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12536.doc diff --git a/changelog.d/12536.doc b/changelog.d/12536.doc new file mode 100644 index 000000000000..4034c42076c4 --- /dev/null +++ b/changelog.d/12536.doc @@ -0,0 +1 @@ +Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. From 70539a5793a733783cb9ff71879dbac5fc2c0b15 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:49:47 +0200 Subject: [PATCH 3/9] Add query for long offline devices --- docs/usage/administration/useful_sql_for_admins.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 8622a7199ac0..3d7076ab5334 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -12,6 +12,7 @@ pg_size_pretty 6420 MB (1 row) ``` + ## Show top 20 larger rooms by state events count ```sql SELECT @@ -266,3 +267,16 @@ WHERE '!YTvKGNlinIzlkMTVRl:matrix.org' ); ``` + +## Show users and devices that have not been online for a while +```sql +SELECT + user_id, + device_id, + user_agent, + TO_TIMESTAMP(last_seen / 1000) AS "last_seen" +FROM + devices +WHERE + last_seen < DATE_PART('epoch', NOW() - INTERVAL '3 month') * 1000; +``` From cb5cbd6b48218364fb932bebd7bdaea7100b9cde Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 26 Apr 2022 08:41:55 +0200 Subject: [PATCH 4/9] Add some note to admin API and database load --- docs/usage/administration/useful_sql_for_admins.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 3d7076ab5334..6e730cf205c3 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -5,6 +5,7 @@ SELECT pg_size_pretty( pg_database_size( 'matrix' ) ); ``` + ### Result example: ``` pg_size_pretty @@ -14,6 +15,10 @@ pg_size_pretty ``` ## Show top 20 larger rooms by state events count +You get the same information when you use the +[admin API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#list-room-api) +and set parameter `order_by=state_events`. + ```sql SELECT r.name, @@ -43,6 +48,7 @@ GROUP BY ORDER BY COUNT(s.room_id) DESC LIMIT 20; ``` + plus same, but with join removed for performance reasons: ```sql SELECT @@ -73,6 +79,7 @@ SELECT FROM ; ``` + ### Result example: ``` state_groups_state - 161687170 @@ -117,6 +124,7 @@ ORDER BY ``` ## Show top 20 users on homeserver by sent events (messages) at last month: +Caution. This query does not use any indexes, can be slow and create load on the database. ```sql SELECT COUNT(*), @@ -178,6 +186,7 @@ WHERE ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 20; ``` + ### Result example: ``` public.state_groups_state - 27 GB @@ -243,6 +252,8 @@ ORDER BY ``` ## Lookup room state info by list of room_id +You get the same information when you use the +[admin API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#room-details-api). ```sql SELECT rss.room_id, From 2244b7312d39d733a9092027b1a5c15932bed64f Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 26 Apr 2022 08:48:16 +0200 Subject: [PATCH 5/9] Move all queries for database information up (togeher) --- .../administration/useful_sql_for_admins.md | 176 +++++++++--------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 6e730cf205c3..102a0657e1b3 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -14,6 +14,94 @@ pg_size_pretty (1 row) ``` +## Show top 20 larger tables by row count +```sql +SELECT + relname, + n_live_tup AS "rows" +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC LIMIT 20; +``` +This query is quick, but may be very approximate, for exact number of rows use: +```sql +SELECT + COUNT(*) +FROM + ; +``` + +### Result example: +``` +state_groups_state - 161687170 +event_auth - 8584785 +event_edges - 6995633 +event_json - 6585916 +event_reference_hashes - 6580990 +events - 6578879 +received_transactions - 5713989 +event_to_state_groups - 4873377 +stream_ordering_to_exterm - 4136285 +current_state_delta_stream - 3770972 +event_search - 3670521 +state_events - 2845082 +room_memberships - 2785854 +cache_invalidation_stream - 2448218 +state_groups - 1255467 +state_group_edges - 1229849 +current_state_events - 1222905 +users_in_public_rooms - 364059 +device_lists_stream - 326903 +user_directory_search - 316433 +``` + +## Show top 20 larger tables by storage size +```sql +SELECT + nspname || '.' || relname AS "relation", + pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" +FROM + pg_class c + LEFT JOIN + pg_namespace n + ON (n.oid = c.relnamespace) +WHERE + nspname NOT IN + ( + 'pg_catalog', + 'information_schema' + ) + AND c.relkind <> 'i' + AND nspname !~ '^pg_toast' +ORDER BY + pg_total_relation_size(c.oid) DESC LIMIT 20; +``` + +### Result example: +``` +public.state_groups_state - 27 GB +public.event_json - 9855 MB +public.events - 3675 MB +public.event_edges - 3404 MB +public.received_transactions - 2745 MB +public.event_reference_hashes - 1864 MB +public.event_auth - 1775 MB +public.stream_ordering_to_exterm - 1663 MB +public.event_search - 1370 MB +public.room_memberships - 1050 MB +public.event_to_state_groups - 948 MB +public.current_state_delta_stream - 711 MB +public.state_events - 611 MB +public.presence_stream - 530 MB +public.current_state_events - 525 MB +public.cache_invalidation_stream - 466 MB +public.receipts_linearized - 279 MB +public.state_groups - 160 MB +public.device_lists_remote_cache - 124 MB +public.state_group_edges - 122 MB +``` + ## Show top 20 larger rooms by state events count You get the same information when you use the [admin API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#list-room-api) @@ -62,48 +150,6 @@ ORDER BY COUNT(s.room_id) DESC LIMIT 20; ``` -## Show top 20 larger tables by row count -```sql -SELECT - relname, - n_live_tup AS "rows" -FROM - pg_stat_user_tables -ORDER BY - n_live_tup DESC LIMIT 20; -``` -This query is quick, but may be very approximate, for exact number of rows use: -```sql -SELECT - COUNT(*) -FROM - ; -``` - -### Result example: -``` -state_groups_state - 161687170 -event_auth - 8584785 -event_edges - 6995633 -event_json - 6585916 -event_reference_hashes - 6580990 -events - 6578879 -received_transactions - 5713989 -event_to_state_groups - 4873377 -stream_ordering_to_exterm - 4136285 -current_state_delta_stream - 3770972 -event_search - 3670521 -state_events - 2845082 -room_memberships - 2785854 -cache_invalidation_stream - 2448218 -state_groups - 1255467 -state_group_edges - 1229849 -current_state_events - 1222905 -users_in_public_rooms - 364059 -device_lists_stream - 326903 -user_directory_search - 316433 -``` - ## Show top 20 rooms by new events count in last 1 day: ```sql SELECT @@ -165,52 +211,6 @@ ORDER BY stream_ordering DESC LIMIT 100; ``` -## Show top 20 larger tables by storage size -```sql -SELECT - nspname || '.' || relname AS "relation", - pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" -FROM - pg_class c - LEFT JOIN - pg_namespace n - ON (n.oid = c.relnamespace) -WHERE - nspname NOT IN - ( - 'pg_catalog', - 'information_schema' - ) - AND c.relkind <> 'i' - AND nspname !~ '^pg_toast' -ORDER BY - pg_total_relation_size(c.oid) DESC LIMIT 20; -``` - -### Result example: -``` -public.state_groups_state - 27 GB -public.event_json - 9855 MB -public.events - 3675 MB -public.event_edges - 3404 MB -public.received_transactions - 2745 MB -public.event_reference_hashes - 1864 MB -public.event_auth - 1775 MB -public.stream_ordering_to_exterm - 1663 MB -public.event_search - 1370 MB -public.room_memberships - 1050 MB -public.event_to_state_groups - 948 MB -public.current_state_delta_stream - 711 MB -public.state_events - 611 MB -public.presence_stream - 530 MB -public.current_state_events - 525 MB -public.cache_invalidation_stream - 466 MB -public.receipts_linearized - 279 MB -public.state_groups - 160 MB -public.device_lists_remote_cache - 124 MB -public.state_group_edges - 122 MB -``` - ## Show rooms with names, sorted by events in this rooms **Sort and order with bash** From 54280b2d326ba2e583a6650cba99ebc3fb69e071 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 26 Apr 2022 14:31:48 +0200 Subject: [PATCH 6/9] Change absolute links to relative links --- docs/usage/administration/useful_sql_for_admins.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 102a0657e1b3..9704b61e97f4 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -104,7 +104,7 @@ public.state_group_edges - 122 MB ## Show top 20 larger rooms by state events count You get the same information when you use the -[admin API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#list-room-api) +[admin API](../../admin_api/rooms.md#list-room-api) and set parameter `order_by=state_events`. ```sql @@ -253,7 +253,7 @@ ORDER BY ## Lookup room state info by list of room_id You get the same information when you use the -[admin API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#room-details-api). +[admin API](../../admin_api/admin_api/rooms.md#room-details-api). ```sql SELECT rss.room_id, From 7374ceea65af9a70025752528374b6b44dcf96f2 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 26 Apr 2022 14:32:35 +0200 Subject: [PATCH 7/9] typo --- docs/usage/administration/useful_sql_for_admins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 9704b61e97f4..04eba8dece3e 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -253,7 +253,7 @@ ORDER BY ## Lookup room state info by list of room_id You get the same information when you use the -[admin API](../../admin_api/admin_api/rooms.md#room-details-api). +[admin API](../../admin_api/rooms.md#room-details-api). ```sql SELECT rss.room_id, From b9071492267c9a858e10771246142c4758da10b5 Mon Sep 17 00:00:00 2001 From: dklimpel <5740567+dklimpel@users.noreply.github.com> Date: Fri, 29 Apr 2022 19:12:01 +0200 Subject: [PATCH 8/9] revert sql formatter --- .../administration/useful_sql_for_admins.md | 225 ++++++------------ 1 file changed, 69 insertions(+), 156 deletions(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 04eba8dece3e..4761f2883f3e 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -2,8 +2,7 @@ ## Size of full matrix db ```sql -SELECT - pg_size_pretty( pg_database_size( 'matrix' ) ); +SELECT pg_size_pretty( pg_database_size( 'matrix' ) ); ``` ### Result example: @@ -16,20 +15,14 @@ pg_size_pretty ## Show top 20 larger tables by row count ```sql -SELECT - relname, - n_live_tup AS "rows" -FROM - pg_stat_user_tables -ORDER BY - n_live_tup DESC LIMIT 20; +SELECT relname, n_live_tup AS "rows" + FROM pg_stat_user_tables + ORDER BY n_live_tup DESC + LIMIT 20; ``` This query is quick, but may be very approximate, for exact number of rows use: ```sql -SELECT - COUNT(*) -FROM - ; +SELECT COUNT(*) FROM ; ``` ### Result example: @@ -58,24 +51,15 @@ user_directory_search - 316433 ## Show top 20 larger tables by storage size ```sql -SELECT - nspname || '.' || relname AS "relation", - pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" -FROM - pg_class c - LEFT JOIN - pg_namespace n - ON (n.oid = c.relnamespace) -WHERE - nspname NOT IN - ( - 'pg_catalog', - 'information_schema' - ) - AND c.relkind <> 'i' - AND nspname !~ '^pg_toast' -ORDER BY - pg_total_relation_size(c.oid) DESC LIMIT 20; +SELECT nspname || '.' || relname AS "relation", + pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" + FROM pg_class c + LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) + WHERE nspname NOT IN ('pg_catalog', 'information_schema') + AND c.relkind <> 'i' + AND nspname !~ '^pg_toast' + ORDER BY pg_total_relation_size(c.oid) DESC + LIMIT 20; ``` ### Result example: @@ -108,107 +92,64 @@ You get the same information when you use the and set parameter `order_by=state_events`. ```sql -SELECT - r.name, - s.room_id, - s.current_state_events -FROM - room_stats_current s - LEFT JOIN - room_stats_state r USING (room_id) -ORDER BY - current_state_events DESC LIMIT 20; +SELECT r.name, s.room_id, s.current_state_events + FROM room_stats_current s + LEFT JOIN room_stats_state r USING (room_id) + ORDER BY current_state_events DESC + LIMIT 20; ``` and by state_group_events count: ```sql -SELECT - rss.name, - s.room_id, - COUNT(s.room_id) -FROM - state_groups_state s - LEFT JOIN - room_stats_state rss USING (room_id) -GROUP BY - s.room_id, - rss.name -ORDER BY - COUNT(s.room_id) DESC LIMIT 20; +SELECT rss.name, s.room_id, COUNT(s.room_id) + FROM state_groups_state s + LEFT JOIN room_stats_state rss USING (room_id) + GROUP BY s.room_id, rss.name + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; ``` plus same, but with join removed for performance reasons: ```sql -SELECT - s.room_id, - COUNT(s.room_id) -FROM - state_groups_state s -GROUP BY - s.room_id -ORDER BY - COUNT(s.room_id) DESC LIMIT 20; +SELECT s.room_id, COUNT(s.room_id) + FROM state_groups_state s + GROUP BY s.room_id + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; ``` ## Show top 20 rooms by new events count in last 1 day: ```sql -SELECT - e.room_id, - r.name, - COUNT(e.event_id) cnt -FROM - events e - LEFT JOIN - room_stats_state r USING (room_id) -WHERE - e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 -GROUP BY - e.room_id, - r.name -ORDER BY - cnt DESC LIMIT 20; +SELECT e.room_id, r.name, COUNT(e.event_id) cnt + FROM events e + LEFT JOIN room_stats_state r USING (room_id) + WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 + GROUP BY e.room_id, r.name + ORDER BY cnt DESC + LIMIT 20; ``` ## Show top 20 users on homeserver by sent events (messages) at last month: Caution. This query does not use any indexes, can be slow and create load on the database. ```sql -SELECT - COUNT(*), - sender -FROM - events -WHERE - ( - type = 'm.room.encrypted' - OR type = 'm.room.message' - ) - AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 -GROUP BY - sender -ORDER BY - COUNT(*) DESC LIMIT 20; +SELECT COUNT(*), sender + FROM events WHERE (type = 'm.room.encrypted' OR type = 'm.room.message') + AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 + GROUP BY sender + ORDER BY COUNT(*) DESC + LIMIT 20; ``` ## Show last 100 messages from needed user, with room names: ```sql -SELECT - e.room_id, - r.name, - e.event_id, - e.type, - e.content, - j.json -FROM - events e - LEFT JOIN - event_json j USING (room_id) - LEFT JOIN - room_stats_state r USING (room_id) -WHERE - sender = '@LOGIN:example.com' - AND e.type = 'm.room.message' -ORDER BY - stream_ordering DESC LIMIT 100; +SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json + FROM events e + LEFT JOIN event_json j USING (room_id) + LEFT JOIN room_stats_state r USING (room_id) + WHERE sender = '@LOGIN:example.com' + AND e.type = 'm.room.message' + ORDER BY stream_ordering DESC + LIMIT 100; ``` ## Show rooms with names, sorted by events in this rooms @@ -223,20 +164,12 @@ Documentation for `psql` command line parameters: https://www.postgresql.org/doc **Sort and order with SQL** ```sql -SELECT - COUNT(*), - event_json.room_id, - room_stats_state.name -FROM - event_json, - room_stats_state -WHERE - room_stats_state.room_id = event_json.room_id -GROUP BY - event_json.room_id, - room_stats_state.name -ORDER BY - COUNT(*) DESC LIMIT 50; +SELECT COUNT(*), event_json.room_id, room_stats_state.name + FROM event_json, room_stats_state + WHERE room_stats_state.room_id = event_json.room_id + GROUP BY event_json.room_id, room_stats_state.name + ORDER BY COUNT(*) DESC + LIMIT 50; ``` ### Result example: @@ -255,39 +188,19 @@ ORDER BY You get the same information when you use the [admin API](../../admin_api/rooms.md#room-details-api). ```sql -SELECT - rss.room_id, - rss.name, - rss.canonical_alias, - rss.topic, - rss.encryption, - rsc.joined_members, - rsc.local_users_in_room, - rss.join_rules -FROM - room_stats_state rss - LEFT JOIN - room_stats_current rsc USING (room_id) -WHERE - room_id IN - ( - WHERE - room_id IN - ( - '!OGEhHVWSdvArJzumhm:matrix.org', - '!YTvKGNlinIzlkMTVRl:matrix.org' - ); +SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, + rsc.joined_members, rsc.local_users_in_room, rss.join_rules + FROM room_stats_state rss + LEFT JOIN room_stats_current rsc USING (room_id) + WHERE room_id IN ( WHERE room_id IN ( + '!OGEhHVWSdvArJzumhm:matrix.org', + '!YTvKGNlinIzlkMTVRl:matrix.org' + ); ``` ## Show users and devices that have not been online for a while ```sql -SELECT - user_id, - device_id, - user_agent, - TO_TIMESTAMP(last_seen / 1000) AS "last_seen" -FROM - devices -WHERE - last_seen < DATE_PART('epoch', NOW() - INTERVAL '3 month') * 1000; +SELECT user_id, device_id, user_agent, TO_TIMESTAMP(last_seen / 1000) AS "last_seen" + FROM devices + WHERE last_seen < DATE_PART('epoch', NOW() - INTERVAL '3 month') * 1000; ``` From 1834ee00ae3f753b86a4f9f5216006c769a8b822 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 3 May 2022 21:23:24 +0200 Subject: [PATCH 9/9] Fix two typos --- docs/usage/administration/useful_sql_for_admins.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index 4761f2883f3e..f3b97f957677 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -16,7 +16,7 @@ pg_size_pretty ## Show top 20 larger tables by row count ```sql SELECT relname, n_live_tup AS "rows" - FROM pg_stat_user_tables + FROM pg_stat_user_tables ORDER BY n_live_tup DESC LIMIT 20; ``` @@ -133,7 +133,8 @@ SELECT e.room_id, r.name, COUNT(e.event_id) cnt Caution. This query does not use any indexes, can be slow and create load on the database. ```sql SELECT COUNT(*), sender - FROM events WHERE (type = 'm.room.encrypted' OR type = 'm.room.message') + FROM events + WHERE (type = 'm.room.encrypted' OR type = 'm.room.message') AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 GROUP BY sender ORDER BY COUNT(*) DESC