Skip to content

Commit

Permalink
task(auth): Clean up redis cache after token pruning.
Browse files Browse the repository at this point in the history
Because:
- Tokens could be orphaned by pruning operations.

This Commit:
- Adds step to to prune-tokens script to proactively remove session tokens in Redis that were just pruned from sql.
- Updates the prune sproc so that the set of affected accounts is relayed back to the token pruner.
- Drops the pruneInterval arg from the prune sproc since it was no longer used now that this happens out of process.
  • Loading branch information
dschom committed Aug 4, 2022
1 parent 7553018 commit 0b57390
Show file tree
Hide file tree
Showing 9 changed files with 423 additions and 127 deletions.
138 changes: 138 additions & 0 deletions packages/db-migrations/databases/fxa/patches/patch-133-134.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
SET NAMES utf8mb4 COLLATE utf8mb4_bin;

-- Make sure there are meta data settings for PrunedUntil and LastPrunedAt
INSERT IGNORE INTO dbMetadata (name, value) VALUES ('prunedUntil', '0');

-- We might have gotten ourselves into a state where data was set to an empty string. If so,
-- start it over from zero.
UPDATE dbMetadata SET value = '0' WHERE name = 'prunedUntil' AND value = '';

-- Update prune to limit total number of sessionTokens examined,
-- and avoid producing the above empty-string bug.
-- maxTokenAge - Any token older than this value will be pruned. A value of 0 denotes that pruning is disabled.
-- maxCodeAge - Any code that was created before now - maxCodeAge will be pruned. A value of 0 denotes pruning is disabled.
-- pruneInterval - The amount of time that must elapse since the previous prune attempt. This guards against inadvertently
-- running a large number of delete operations in succession.
-- unblockCodesDeleted - Number of unblock codes deleted
-- signInCodesDeleted - Number of sign in codes deleted
-- accountResetTokensDeleted - Number of acount reset tokens deleted
-- passwordForgotTokensDeleted - Number of password forgot tokens deleted
-- passwordChangeTokensDeleted - Number of password change tokens deleted
-- sessionTokensDeleted - Number of session tokens deleted deleted
CREATE PROCEDURE `prune_9` (
IN `curTime` BIGINT UNSIGNED,
IN `maxTokenAge` BIGINT UNSIGNED,
IN `maxCodeAge` BIGINT UNSIGNED,
OUT `unblockCodesDeleted` INT UNSIGNED,
OUT `signInCodesDeleted` INT UNSIGNED,
OUT `accountResetTokensDeleted` INT UNSIGNED,
OUT `passwordForgotTokensDeleted` INT UNSIGNED,
OUT `passwordChangeTokensDeleted` INT UNSIGNED,
OUT `sessionTokensDeleted` INT UNSIGNED
)
BEGIN
DECLARE EXIT HANDLER FOR SQLEXCEPTION
BEGIN
ROLLBACK;
RESIGNAL;
END;

SET unblockCodesDeleted = 0;
SET signInCodesDeleted = 0;
SET accountResetTokensDeleted = 0;
SET passwordForgotTokensDeleted = 0;
SET passwordChangeTokensDeleted = 0;
SET sessionTokensDeleted = 0;

SELECT @lockAcquired := GET_LOCK('fxa-auth-server.prune-lock', 3);
IF @lockAcquired THEN

IF maxCodeAge > 0 THEN
DELETE FROM unblockCodes WHERE createdAt < curTime - maxCodeAge ORDER BY createdAt LIMIT 10000;
SET unblockCodesDeleted = ROW_COUNT();
DELETE FROM signinCodes WHERE createdAt < curTime - maxCodeAge ORDER BY createdAt LIMIT 10000;
SET signInCodesDeleted = ROW_COUNT();
END IF;

IF maxTokenAge > 0 THEN
DELETE FROM accountResetTokens WHERE createdAt < curTime - maxTokenAge ORDER BY createdAt LIMIT 10000;
SET accountResetTokensDeleted = ROW_COUNT();
DELETE FROM passwordForgotTokens WHERE createdAt < curTime - maxTokenAge ORDER BY createdAt LIMIT 10000;
SET passwordForgotTokensDeleted = ROW_COUNT();
DELETE FROM passwordChangeTokens WHERE createdAt < curTime - maxTokenAge ORDER BY createdAt LIMIT 10000;
SET passwordChangeTokensDeleted = ROW_COUNT();

-- Pruning session tokens is complicated because:
-- * we can't prune them if there is an associated device record, and
-- * we have to delete from both sessionTokens and unverifiedTokens tables, and
-- * MySQL won't allow `LIMIT` to be used in a multi-table delete.
-- To achieve all this in an efficient manner, we prune tokens within a specific
-- time window rather than using a `LIMIT` clause. At the end of each run we
-- record the new lower-bound on creation time for tokens that might have expired.
START TRANSACTION;

-- Step 1: Find out how far we got on previous iterations.
SELECT @pruneFrom := value FROM dbMetadata WHERE name = 'prunedUntil';

-- Step 2: Calculate what timestamp we will reach on this iteration
-- if we purge a sensibly-sized batch of tokens.
-- N.B. We deliberately do not filter on whether the token has
-- a device here. We want to limit the number of tokens that we
-- *examine*, regardless of whether it actually delete them.
SELECT @pruneUntil := MAX(createdAt) FROM (
SELECT createdAt FROM sessionTokens
WHERE createdAt >= @pruneFrom AND createdAt < curTime - maxTokenAge
ORDER BY createdAt
LIMIT 10000
) AS candidatesForPruning;

-- This will be NULL if there are no expired tokens,
-- in which case we have nothing to do.
IF @pruneUntil IS NOT NULL THEN

-- Step 3: relay impacted accounts back to token pruner
SELECT distinct st.uid
from sessionTokens AS st
LEFT JOIN unverifiedTokens AS ut
ON st.tokenId = ut.tokenId
WHERE st.createdAt > @pruneFrom
AND st.createdAt <= @pruneUntil
AND NOT EXISTS (
SELECT sessionTokenId FROM devices
WHERE uid = st.uid AND sessionTokenId = st.tokenId
);

-- Step 4: Prune sessionTokens and unverifiedTokens tables.
-- Here we *do* filter on whether a device record exists.
-- We might not actually delete any tokens, but we will definitely
-- be able to increase 'prunedUntil' for the next run.
DELETE st, ut
FROM sessionTokens AS st
LEFT JOIN unverifiedTokens AS ut
ON st.tokenId = ut.tokenId
WHERE st.createdAt > @pruneFrom
AND st.createdAt <= @pruneUntil
AND NOT EXISTS (
SELECT sessionTokenId FROM devices
WHERE uid = st.uid AND sessionTokenId = st.tokenId
);

SET sessionTokensDeleted = ROW_COUNT();

-- Step 4: Tell following iterations how far we got.
UPDATE dbMetadata
SET value = @pruneUntil
WHERE name = 'prunedUntil';

END IF;

END IF;

COMMIT;
SELECT RELEASE_LOCK('fxa-auth-server.prune-lock');

END IF;

END;

UPDATE dbMetadata SET value = '134' WHERE name = 'schema-patch-level';
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
-- SET NAMES utf8mb4 COLLATE utf8mb4_bin;

-- DROP PROCEDURE `prune_9`;

-- UPDATE dbMetadata SET value = '133' WHERE name = 'schema-patch-level';
2 changes: 1 addition & 1 deletion packages/db-migrations/databases/fxa/target-patch.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
"level": 133
"level": 134
}
101 changes: 81 additions & 20 deletions packages/fxa-auth-server/scripts/prune-tokens.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import program from 'commander';
import { setupDatabase } from 'fxa-shared/db';
import { BaseAuthModel } from 'fxa-shared/db/models/auth';
import { StatsD } from 'hot-shots';
import { RedisShared } from 'fxa-shared/db/redis';
import moment from 'moment';
import { SessionToken } from 'fxa-shared/db/models/auth/session-token';
const { PruneTokens } = require('fxa-shared/db/models/auth');
const pckg = require('../package.json');

Expand All @@ -25,7 +25,13 @@ export async function init() {
const config = require('../config').getProperties();
const statsd = new StatsD(config.statsd);
const log = require('../lib/log')(config.log.level, 'prune-tokens', statsd);
const redis = new RedisShared(config.redis, log, statsd);
const redis = require('../lib/redis')(
{
...config.redis,
...config.redis.sessionTokens,
},
log
);

// Parse args
program
Expand Down Expand Up @@ -84,6 +90,7 @@ Exit Codes:

// Create token pruner instance
const pruner = new PruneTokens(statsd, log);
const accountsImpacted = new Set<string>();

// Start max session pruning
if (maxSessions > 0) {
Expand All @@ -97,24 +104,9 @@ Exit Codes:

// Any account impacted by the prune operation should have it's session tokens cleaned out
// of the redis cache.
for (const row of result.results?.rows[1] || []) {
for (const row of result.uids || []) {
const uid = row.uid.toString('hex');
log.info('try flushing redis cache', { uid });

// Check for accounts with potentially orphaned session tokens, and try to flush them out
// of the redis cache.
const tokens = await redis.getSessionTokens(uid);

if (tokens) {
const tokenIds = Object.keys(tokens);
await redis.pruneSessionTokens(row.uid, tokenIds);
log.info('flushed redis cache tokens', {
uid,
tokenCount: tokenIds.length,
});
} else {
log.info('no tokens found in redis cache', { uid });
}
accountsImpacted.add(uid);
}
} catch (err) {
log.error('error during limit sessions', err);
Expand All @@ -134,7 +126,12 @@ Exit Codes:
codeMaxAge: codeMaxAge + 'ms',
});
const result = await pruner.prune(tokenMaxAge, codeMaxAge);
log.info('token pruning complete', result);
log.info('token pruning complete', result.outputs);

for (const row of result.uids || []) {
const uid = row.uid.toString('hex');
accountsImpacted.add(uid);
}
} catch (err) {
log.error('error during token prune', err);
return 3;
Expand All @@ -145,6 +142,70 @@ Exit Codes:
);
}

// Clean up redis cache
if (accountsImpacted.size > 0) {
for (const uid of [...accountsImpacted]) {
try {
// Pull session tokens from sql db.
const sessionTokens = await SessionToken.findByUid(uid);
const sqlSessionTokenIds = new Set(sessionTokens.map((x) => x.tokenId));

// Short circuit if there are no sql session tokens
if (sqlSessionTokenIds.size === 0) {
log.info('no tokens found in sql db', { uid });
continue;
}

// Pull session tokens from redis, sanity check sizes
const redisSessionTokens = await redis.getSessionTokens(uid);
const redisSessionTokenIds = new Set(
Object.keys(redisSessionTokens || {})
);

// Short circuit if there are no redis session tokens
if (redisSessionTokenIds.size === 0) {
log.info('no tokens found in redis cache', { uid });
continue;
}

// Sanity check size of sqlSessionTokenIds and redisSessionTokenIds
if (maxSessions > 0 && sqlSessionTokenIds.size > maxSessions) {
log.warn(
`Found ${sqlSessionTokenIds.size}. Expected no more than ${maxSessions}.`
);
}
if (redisSessionTokenIds.size > 500) {
log.warn(
`found ${redisSessionTokenIds.size}. expected no more than 500.`
);
}

// Difference redisSessionTokenIds and sqlSessionTokenIds. This will result in a
// set of orphaned tokens.
const difference = new Set(redisSessionTokenIds);
for (const x of sqlSessionTokenIds) {
difference.delete(x);
}

// Delete orphaned tokens from Redis
const orphanedTokenIds = [...difference];
if (orphanedTokenIds.length > 0) {
log.info(`pruning orphaned tokens from redis`, {
uid,
count: orphanedTokenIds.length,
});
await redis.pruneSessionTokens(uid, orphanedTokenIds);
} else {
log.info(`no orphaned tokens found in redis`);
}
} catch (err) {
log.err(`error while pruning redis cache for account ${uid}`, err);
}
}
} else {
log.info('no accounts impacted. skipping redis cache clean up.');
}

return 0;
}

Expand Down
Loading

0 comments on commit 0b57390

Please sign in to comment.