diff --git a/packages/fxa-auth-server/lib/payments/paypal/processor.ts b/packages/fxa-auth-server/lib/payments/paypal/processor.ts index 87ba8b719dc..0dd389dfe4a 100644 --- a/packages/fxa-auth-server/lib/payments/paypal/processor.ts +++ b/packages/fxa-auth-server/lib/payments/paypal/processor.ts @@ -350,7 +350,7 @@ export class PaypalProcessor { return; } - public async processInvoices() { + public async *processInvoices() { // Generate a time `invoiceAge` hours prior. const invoiceAgeInSeconds = hoursBeforeInSeconds(this.invoiceAge); @@ -369,6 +369,8 @@ export class PaypalProcessor { }); reportSentryError(err); } + + yield; } } } diff --git a/packages/fxa-auth-server/package.json b/packages/fxa-auth-server/package.json index 5335453ff8d..695f15f95ec 100644 --- a/packages/fxa-auth-server/package.json +++ b/packages/fxa-auth-server/package.json @@ -113,6 +113,7 @@ "poolee": "^1.0.1", "punycode.js": "2.1.0", "qrcode": "^1.5.0", + "redlock": "^5.0.0-beta.2", "request": "^2.88.2", "safe-regex": "^2.1.1", "safe-url-assembler": "1.3.5", diff --git a/packages/fxa-auth-server/scripts/paypal-processor.ts b/packages/fxa-auth-server/scripts/paypal-processor.ts index 7507df708de..5b9ff5c7e52 100644 --- a/packages/fxa-auth-server/scripts/paypal-processor.ts +++ b/packages/fxa-auth-server/scripts/paypal-processor.ts @@ -3,6 +3,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import program from 'commander'; import { StatsD } from 'hot-shots'; +import Redis from 'ioredis'; +import Redlock, { Lock } from 'redlock'; import Container from 'typedi'; import { promisify } from 'util'; @@ -13,6 +15,21 @@ import { setupProcessingTaskObjects } from '../lib/payments/processing-tasks-set const pckg = require('../package.json'); const config = require('../config').getProperties(); +const PAYPAL_PROCESSOR_LOCK = 'fxa-paypal-processor-lock'; +const DEFAULT_LOCK_DURATION_MS = 300000; +let lock: Lock; + +const initTimer = () => { + let start = Date.now(); + + const reset = () => (start = Date.now()); + const elapsed = () => Date.now() - start; + + return { + reset, + elapsed, + }; +}; export async function init() { // Load program options @@ -29,8 +46,28 @@ export async function init() { 'How old in hours the invoice must be to get processed. Defaults to 6.', '6' ) + .option( + '-l, --use-lock [bool]', + 'Whether to require a distributed lock to run. Use "false" to disable. Defaults to true.', + true + ) + .option( + '-n, --lock-name [name]', + `The name of the resource for which to acquire a distributed lock. Defaults to ${PAYPAL_PROCESSOR_LOCK}.`, + PAYPAL_PROCESSOR_LOCK + ) + .option( + '-d, --lock-duration [milliseconds]', + `The max duration in milliseconds to hold the lock. The lock will be extended as needed. Defaults to ${DEFAULT_LOCK_DURATION_MS}.`, + DEFAULT_LOCK_DURATION_MS + ) .parse(process.argv); + // every arg is a string + const useLock = program.useLock !== 'false'; + const lockDuration = + parseInt(`${program.lockDuration}`) || DEFAULT_LOCK_DURATION_MS; + const { log, database, senders } = await setupProcessingTaskObjects( 'paypal-processor' ); @@ -53,17 +90,43 @@ export async function init() { ); const statsd = Container.get(StatsD); statsd.increment('paypal-processor.startup'); - await processor.processInvoices(); + + const timer = initTimer(); + + if (useLock) { + try { + const redis = new Redis(config.redis); + const redlock = new Redlock([redis], { retryCount: 1 }); + lock = await redlock.acquire([program.lockName], lockDuration); + } catch (err) { + throw new Error(`Cannot acquire lock to run: ${err.message}`); + } + } + + for await (const _ of processor.processInvoices()) { + if (useLock && timer.elapsed() > Math.floor(lockDuration / 2)) { + await lock.extend(timer.elapsed()); + timer.reset(); + } + } + statsd.increment('paypal-processor.shutdown'); await promisify(statsd.close).bind(statsd)(); return 0; } if (require.main === module) { + let exitStatus = 1; init() + .then((result) => { + exitStatus = result; + }) .catch((err) => { console.error(err); - process.exit(1); + exitStatus = 1; }) - .then((result) => process.exit(result)); + .finally(() => { + lock?.release(); + process.exit(exitStatus); + }); } diff --git a/packages/fxa-auth-server/test/local/payments/paypal-processor.js b/packages/fxa-auth-server/test/local/payments/paypal-processor.js index a5f2f16f66a..da06b6ce98d 100644 --- a/packages/fxa-auth-server/test/local/payments/paypal-processor.js +++ b/packages/fxa-auth-server/test/local/payments/paypal-processor.js @@ -605,7 +605,11 @@ describe('PaypalProcessor', () => { yield invoice; }, }); - await processor.processInvoices(); + // eslint-disable-next-line + for await (const _ of processor.processInvoices()) { + // No value yield'd; yielding control for potential distributed lock + // extension in actual use case + } sinon.assert.calledOnceWithExactly( mockLog.info, 'processInvoice.processing', @@ -628,7 +632,11 @@ describe('PaypalProcessor', () => { }, }); try { - await processor.processInvoices(); + // eslint-disable-next-line + for await (const _ of processor.processInvoices()) { + // No value yield'd; yielding control for potential distributed lock + // extension in actual use case + } assert.fail('Process invoicce should fail'); } catch (err) { sinon.assert.calledOnceWithExactly( diff --git a/yarn.lock b/yarn.lock index 251a860498e..37aa448ba7d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22274,6 +22274,7 @@ fsevents@~2.1.1: punycode.js: 2.1.0 qrcode: ^1.5.0 read: 1.0.7 + redlock: ^5.0.0-beta.2 request: ^2.88.2 rimraf: ^3.0.2 safe-regex: ^2.1.1 @@ -32251,6 +32252,13 @@ fsevents@~2.1.1: languageName: node linkType: hard +"node-abort-controller@npm:^3.0.1": + version: 3.0.1 + resolution: "node-abort-controller@npm:3.0.1" + checksum: 2b3d75c65249fea99e8ba22da3a8bc553f034f44dd12f5f4b38b520f718b01c88718c978f0c24c2a460fc01de9a80b567028f547b94440cb47adeacfeb82c2ee + languageName: node + linkType: hard + "node-addon-api@npm:^4.3.0": version: 4.3.0 resolution: "node-addon-api@npm:4.3.0" @@ -37452,6 +37460,15 @@ fsevents@~2.1.1: languageName: node linkType: hard +"redlock@npm:^5.0.0-beta.2": + version: 5.0.0-beta2 + resolution: "redlock@npm:5.0.0-beta2" + dependencies: + node-abort-controller: ^3.0.1 + checksum: d8a0d6d472922d146077e3c12946b942108e3041439fdadced79c3dc285ec3d509a48cee33f7da125e71b3868c65ba248b612fb65825aacfa5e67c118e1ba543 + languageName: node + linkType: hard + "reduce-css-calc@npm:^2.1.6, reduce-css-calc@npm:^2.1.8": version: 2.1.8 resolution: "reduce-css-calc@npm:2.1.8"