From 58f2b9f6a67c394ff4bc0795524ae5d93259e577 Mon Sep 17 00:00:00 2001 From: Aaditya Sondhi Date: Mon, 26 Sep 2022 16:08:17 -0400 Subject: [PATCH] server: fix bottlenecks when purging system.web_sessions The previous purging limit was found to be too low which resulted in the `system.web_sessions` table growing rapidly. This happened when a customer-side automation script was generating a large number of sessions. Additionally, we were using `ORDER BY random()` when deleting expired sessions from the table which caused a full table scan of a large table. This resulted in high cpu usage on every purge attempt with only very few rows actually being deleted. Over time, this turns into a nonideal feedback loop. This change fixes the two issues by: 1. Increasing the defauilt purging limit to 1000 from 10. 2. Remove the `ORDER BY random()` when purging the table to avoid full table scans on every purge. support ticket: https://github.com/cockroachlabs/support/issues/1812 resolves #88622 Release note: None --- docs/generated/settings/settings-for-tenants.txt | 2 +- docs/generated/settings/settings.html | 2 +- pkg/server/purge_auth_session.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index 677f232bfdbd..30104d5444da 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -87,7 +87,7 @@ server.user_login.password_hashes.default_cost.scram_sha_256 integer 119680 the server.user_login.timeout duration 10s timeout after which client authentication times out if some system range is unavailable (0 = no timeout) server.user_login.upgrade_bcrypt_stored_passwords_to_scram.enabled boolean true whether to automatically re-encode stored passwords using crdb-bcrypt to scram-sha-256 server.web_session.auto_logout.timeout duration 168h0m0s the duration that web sessions will survive before being periodically purged, since they were last used -server.web_session.purge.max_deletions_per_cycle integer 10 the maximum number of old sessions to delete for each purge +server.web_session.purge.max_deletions_per_cycle integer 1000 the maximum number of old sessions to delete for each purge server.web_session.purge.period duration 1h0m0s the time until old sessions are deleted server.web_session.purge.ttl duration 1h0m0s if nonzero, entries in system.web_sessions older than this duration are periodically purged server.web_session_timeout duration 168h0m0s the duration that a newly created web session will be valid diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index 9ae6e6652691..0c8f70500900 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -106,7 +106,7 @@ server.user_login.timeoutduration10stimeout after which client authentication times out if some system range is unavailable (0 = no timeout) server.user_login.upgrade_bcrypt_stored_passwords_to_scram.enabledbooleantruewhether to automatically re-encode stored passwords using crdb-bcrypt to scram-sha-256 server.web_session.auto_logout.timeoutduration168h0m0sthe duration that web sessions will survive before being periodically purged, since they were last used -server.web_session.purge.max_deletions_per_cycleinteger10the maximum number of old sessions to delete for each purge +server.web_session.purge.max_deletions_per_cycleinteger1000the maximum number of old sessions to delete for each purge server.web_session.purge.periodduration1h0m0sthe time until old sessions are deleted server.web_session.purge.ttlduration1h0m0sif nonzero, entries in system.web_sessions older than this duration are periodically purged server.web_session_timeoutduration168h0m0sthe duration that a newly created web session will be valid diff --git a/pkg/server/purge_auth_session.go b/pkg/server/purge_auth_session.go index da0e846a4714..a909624191c2 100644 --- a/pkg/server/purge_auth_session.go +++ b/pkg/server/purge_auth_session.go @@ -50,7 +50,7 @@ var ( settings.TenantWritable, "server.web_session.purge.max_deletions_per_cycle", "the maximum number of old sessions to delete for each purge", - 10, + 1000, ).WithPublic() ) @@ -90,21 +90,21 @@ func (s *authenticationServer) purgeOldSessions(ctx context.Context) { deleteOldExpiredSessionsStmt = ` DELETE FROM system.web_sessions WHERE "expiresAt" < $1 -ORDER BY random() +ORDER BY "expiresAt" ASC LIMIT $2 RETURNING 1 ` deleteOldRevokedSessionsStmt = ` DELETE FROM system.web_sessions WHERE "revokedAt" < $1 -ORDER BY random() +ORDER BY "expiresAt" ASC LIMIT $2 RETURNING 1 ` deleteSessionsAutoLogoutStmt = ` DELETE FROM system.web_sessions WHERE "lastUsedAt" < $1 -ORDER BY random() +ORDER BY "expiresAt" ASC LIMIT $2 RETURNING 1 `