From 9ede1f7106b535326e0bf2fdc06a61c35f7f4cfe Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 13 Dec 2023 10:40:21 -0500 Subject: [PATCH 01/16] timestamp repo --- lib/constants.ts | 2 + lib/providers/circuit-breaker/s3.ts | 6 +-- lib/repositories/base.ts | 4 ++ lib/repositories/timestamp-repository.ts | 55 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 lib/repositories/timestamp-repository.ts diff --git a/lib/constants.ts b/lib/constants.ts index 57bcf5eb..1714a23e 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -12,6 +12,7 @@ export const BETA_COMPLIANCE_S3_KEY = 'beta.json'; export const DYNAMO_TABLE_NAME = { FADES: 'Fades', SYNTHETIC_SWITCH_TABLE: 'SyntheticSwitchTable', + TIMESTAMP: 'Timestamp', }; export const DYNAMO_TABLE_KEY = { @@ -23,4 +24,5 @@ export const DYNAMO_TABLE_KEY = { TRADE_TYPE: 'type', LOWER: 'lower', ENABLED: 'enabled', + TIMESTAMP: 'timestamp', }; diff --git a/lib/providers/circuit-breaker/s3.ts b/lib/providers/circuit-breaker/s3.ts index 00d56385..dc03b749 100644 --- a/lib/providers/circuit-breaker/s3.ts +++ b/lib/providers/circuit-breaker/s3.ts @@ -3,9 +3,9 @@ import { NodeHttpHandler } from '@smithy/node-http-handler'; import { MetricsLogger, Unit } from 'aws-embedded-metrics'; import Logger from 'bunyan'; +import { CircuitBreakerConfiguration, CircuitBreakerConfigurationProvider } from '.'; import { Metric } from '../../entities'; import { checkDefined } from '../../preconditions/preconditions'; -import { CircuitBreakerConfiguration, CircuitBreakerConfigurationProvider } from '.'; export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConfigurationProvider { private log: Logger; @@ -13,8 +13,8 @@ export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConf private lastUpdatedTimestamp: number; private client: S3Client; - // try to refetch endpoints every 5 mins - private static UPDATE_PERIOD_MS = 5 * 60000; + // try to refetch endpoints every minute + private static UPDATE_PERIOD_MS = 1 * 60000; private static FILL_RATE_THRESHOLD = 0.75; constructor(_log: Logger, private bucket: string, private key: string) { diff --git a/lib/repositories/base.ts b/lib/repositories/base.ts index 277246ab..eeed8c3e 100644 --- a/lib/repositories/base.ts +++ b/lib/repositories/base.ts @@ -65,3 +65,7 @@ export interface BaseSwitchRepository { putSynthSwitch(trade: SynthSwitchTrade, lower: string, enabled: boolean): Promise; syntheticQuoteForTradeEnabled(trade: SynthSwitchQueryParams): Promise; } + +export interface BaseTimestampRepository { + updateTimestamp(hash: string, ts: number): Promise; +} diff --git a/lib/repositories/timestamp-repository.ts b/lib/repositories/timestamp-repository.ts new file mode 100644 index 00000000..142a5b41 --- /dev/null +++ b/lib/repositories/timestamp-repository.ts @@ -0,0 +1,55 @@ +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import Logger from 'bunyan'; +import { Entity, Table } from 'dynamodb-toolbox'; + +import { DYNAMO_TABLE_KEY, DYNAMO_TABLE_NAME } from '../constants'; +import { BaseTimestampRepository } from './base'; + +export class TimestampRepository implements BaseTimestampRepository { + static log: Logger; + static PARTITION_KEY = 'hash'; + + static create(documentClient: DynamoDBDocumentClient): BaseTimestampRepository { + this.log = Logger.createLogger({ + name: 'DynamoTimestampRepository', + serializers: Logger.stdSerializers, + }); + + const table = new Table({ + name: DYNAMO_TABLE_NAME.TIMESTAMP, + partitionKey: TimestampRepository.PARTITION_KEY, + DocumentClient: documentClient, + }); + + const entity = new Entity({ + name: 'SynthSwitchEntity', + attributes: { + [TimestampRepository.PARTITION_KEY]: { partitionKey: true }, + [`${DYNAMO_TABLE_KEY.TIMESTAMP}`]: { type: 'number' }, + }, + table: table, + autoExecute: true, + } as const); + + return new TimestampRepository(table, entity); + } + + private constructor( + // eslint-disable-next-line + // @ts-expect-error + private readonly _switchTable: Table<'Timestamp', 'hash', null>, + private readonly entity: Entity + ) {} + + public async updateTimestamp(hash: string, ts: number): Promise { + await this.entity.put( + { + [TimestampRepository.PARTITION_KEY]: hash, + [`${DYNAMO_TABLE_KEY.TIMESTAMP}`]: ts, + }, + { + execute: true, + } + ); + } +} From 568eab1cf0285343d34cd29e853dfe6a3b8e0e88 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Thu, 14 Dec 2023 10:20:18 -0500 Subject: [PATCH 02/16] dynamo timestamp repo --- jest-dynamodb-config.js | 10 +++ lib/constants.ts | 4 +- lib/cron/fade-rate.ts | 34 ++++++-- lib/providers/circuit-breaker/s3.ts | 15 +++- lib/quoters/WebhookQuoter.ts | 87 +++++++++---------- lib/repositories/base.ts | 15 +++- lib/repositories/fades-repository.ts | 27 +++--- lib/repositories/timestamp-repository.ts | 67 +++++++++++--- test/repositories/shared.ts | 10 +++ test/repositories/switch-repository.test.ts | 22 ++--- .../repositories/timestamp-repository.test.ts | 54 ++++++++++++ 11 files changed, 249 insertions(+), 96 deletions(-) create mode 100644 test/repositories/shared.ts create mode 100644 test/repositories/timestamp-repository.test.ts diff --git a/jest-dynamodb-config.js b/jest-dynamodb-config.js index 532872f4..d76b13c7 100644 --- a/jest-dynamodb-config.js +++ b/jest-dynamodb-config.js @@ -10,6 +10,16 @@ module.exports = { ], ProvisionedThroughput: { ReadCapacityUnits: 10, WriteCapacityUnits: 10 }, }, + { + TableName: 'Timestamp', + KeySchema: [ + { AttributeName: 'hash', KeyType: 'HASH' }, + ], + AttributeDefinitions: [ + { AttributeName: 'hash', AttributeType: 'S' }, + ], + ProvisionedThroughput: { ReadCapacityUnits: 10, WriteCapacityUnits: 10 }, + } ], port: 8000, }; diff --git a/lib/constants.ts b/lib/constants.ts index 1714a23e..3dcc2c47 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -24,5 +24,7 @@ export const DYNAMO_TABLE_KEY = { TRADE_TYPE: 'type', LOWER: 'lower', ENABLED: 'enabled', - TIMESTAMP: 'timestamp', + BLOCK_UNTIL_TIMESTAMP: 'blockUntilTimestamp', + LAST_POST_TIMESTAMP: 'lastPostTimestamp', + FADED: 'faded', }; diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 1ceddbd0..631fc083 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -1,3 +1,5 @@ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { metricScope, MetricsLogger } from 'aws-embedded-metrics'; import { ScheduledHandler } from 'aws-lambda/trigger/cloudwatch-events'; import { EventBridgeEvent } from 'aws-lambda/trigger/eventbridge'; @@ -7,6 +9,7 @@ import { BETA_S3_KEY, FADE_RATE_BUCKET, FADE_RATE_S3_KEY, + FILL_RATE_THRESHOLD, PRODUCTION_S3_KEY, WEBHOOK_CONFIG_BUCKET, } from '../constants'; @@ -14,7 +17,8 @@ import { CircuitBreakerMetricDimension } from '../entities'; import { checkDefined } from '../preconditions/preconditions'; import { S3WebhookConfigurationProvider } from '../providers'; import { S3CircuitBreakerConfigurationProvider } from '../providers/circuit-breaker/s3'; -import { FadesRepository, FadesRowType, SharedConfigs } from '../repositories'; +import { BaseTimestampRepository, FadesRepository, FadesRowType, SharedConfigs } from '../repositories'; +import { TimestampRepository } from '../repositories/timestamp-repository'; import { STAGE } from '../util/stage'; export const handler: ScheduledHandler = metricScope((metrics) => async (_event: EventBridgeEvent) => { @@ -32,6 +36,15 @@ async function main(metrics: MetricsLogger) { const stage = process.env['stage']; const s3Key = stage === STAGE.BETA ? BETA_S3_KEY : PRODUCTION_S3_KEY; const webhookProvider = new S3WebhookConfigurationProvider(log, `${WEBHOOK_CONFIG_BUCKET}-${stage}-1`, s3Key); + const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient({}), { + marshallOptions: { + convertEmptyValues: true, + }, + unmarshallOptions: { + wrapNumbers: true, + }, + }); + const timestampDB = TimestampRepository.create(documentClient); const sharedConfig: SharedConfigs = { Database: checkDefined(process.env.REDSHIFT_DATABASE), @@ -44,18 +57,23 @@ async function main(metrics: MetricsLogger) { if (result) { const addressToFillerHash = await webhookProvider.addressToFillerHash(); - const fillerFadeRate = calculateFillerFadeRates(result, addressToFillerHash, log); + const fillersNewFades = calculateFillerFadeRates(result, addressToFillerHash, log); log.info({ fadeRates: [...fillerFadeRate.entries()] }, 'filler fade rate'); - const configProvider = new S3CircuitBreakerConfigurationProvider( - log, - `${FADE_RATE_BUCKET}-prod-1`, - FADE_RATE_S3_KEY - ); - await configProvider.putConfigurations(fillerFadeRate, metrics); + const toUpdate = [...fillerFadeRate.entries()].filter(([, rate]) => rate >= FILL_RATE_THRESHOLD); + log.info({ toUpdate }, 'filler for which to update timestamp'); + await timestampDB.updateTimestampsBatch(toUpdate, Math.floor(Date.now() / 1000)); } } +export function getFillersNewFades( + rows: FadesRowType[], + addressToFillerHash: Map, + log?: Logger +): Map { + const; +} + // aggregates potentially multiple filler addresses into filler name // and calculates fade rate for each export function calculateFillerFadeRates( diff --git a/lib/providers/circuit-breaker/s3.ts b/lib/providers/circuit-breaker/s3.ts index dc03b749..f2ea21fe 100644 --- a/lib/providers/circuit-breaker/s3.ts +++ b/lib/providers/circuit-breaker/s3.ts @@ -3,19 +3,23 @@ import { NodeHttpHandler } from '@smithy/node-http-handler'; import { MetricsLogger, Unit } from 'aws-embedded-metrics'; import Logger from 'bunyan'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { CircuitBreakerConfiguration, CircuitBreakerConfigurationProvider } from '.'; import { Metric } from '../../entities'; import { checkDefined } from '../../preconditions/preconditions'; +import { BaseTimestampRepository } from '../../repositories'; +import { TimestampRepository } from '../../repositories/timestamp-repository'; export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConfigurationProvider { private log: Logger; private fillers: CircuitBreakerConfiguration[]; private lastUpdatedTimestamp: number; private client: S3Client; + private timestampDB: BaseTimestampRepository; // try to refetch endpoints every minute private static UPDATE_PERIOD_MS = 1 * 60000; - private static FILL_RATE_THRESHOLD = 0.75; constructor(_log: Logger, private bucket: string, private key: string) { this.log = _log.child({ quoter: 'S3CircuitBreakerConfigurationProvider' }); @@ -26,6 +30,15 @@ export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConf requestTimeout: 500, }), }); + const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient({}), { + marshallOptions: { + convertEmptyValues: true, + }, + unmarshallOptions: { + wrapNumbers: true, + }, + }); + this.timestampDB = TimestampRepository.create(documentClient); } async getConfigurations(): Promise { diff --git a/lib/quoters/WebhookQuoter.ts b/lib/quoters/WebhookQuoter.ts index 8f6ec069..05e1598e 100644 --- a/lib/quoters/WebhookQuoter.ts +++ b/lib/quoters/WebhookQuoter.ts @@ -4,12 +4,20 @@ import axios, { AxiosError, AxiosResponse } from 'axios'; import Logger from 'bunyan'; import { v4 as uuidv4 } from 'uuid'; -import { FirehoseLogger } from '../providers/analytics'; -import { Metric, metricContext, QuoteRequest, QuoteResponse, AnalyticsEvent, AnalyticsEventType, WebhookResponseType } from '../entities'; +import { Quoter, QuoterType } from '.'; +import { + AnalyticsEvent, + AnalyticsEventType, + Metric, + metricContext, + QuoteRequest, + QuoteResponse, + WebhookResponseType, +} from '../entities'; import { WebhookConfiguration, WebhookConfigurationProvider } from '../providers'; +import { FirehoseLogger } from '../providers/analytics'; import { CircuitBreakerConfigurationProvider } from '../providers/circuit-breaker'; import { FillerComplianceConfigurationProvider } from '../providers/compliance'; -import { Quoter, QuoterType } from '.'; import { timestampInMstoISOString } from '../util/time'; // TODO: shorten, maybe take from env config @@ -19,28 +27,26 @@ const WEBHOOK_TIMEOUT_MS = 500; // endpoints must return well-formed QuoteResponse JSON export class WebhookQuoter implements Quoter { private log: Logger; - private readonly ALLOW_LIST: Set; constructor( _log: Logger, private firehose: FirehoseLogger, private webhookProvider: WebhookConfigurationProvider, private circuitBreakerProvider: CircuitBreakerConfigurationProvider, - private complianceProvider: FillerComplianceConfigurationProvider, - _allow_list: Set = new Set(['1fcc7006edf742beb4052907a6bf1631e18f3d0793e5901a14a485ec59ee451d']) + private complianceProvider: FillerComplianceConfigurationProvider ) { this.log = _log.child({ quoter: 'WebhookQuoter' }); - this.ALLOW_LIST = _allow_list; } public async quote(request: QuoteRequest): Promise { const endpoints = await this.getEligibleEndpoints(); const endpointToAddrsMap = await this.complianceProvider.getEndpointToExcludedAddrsMap(); endpoints.filter((e) => { - return endpointToAddrsMap.get(e.endpoint) === undefined || - !endpointToAddrsMap.get(e.endpoint)?.has(request.swapper); + return ( + endpointToAddrsMap.get(e.endpoint) === undefined || !endpointToAddrsMap.get(e.endpoint)?.has(request.swapper) + ); }); - + this.log.info({ endpoints }, `Fetching quotes from ${endpoints.length} endpoints`); const quotes = await Promise.all(endpoints.map((e) => this.fetchQuote(e, request))); return quotes.filter((q) => q !== null) as QuoteResponse[]; @@ -63,7 +69,6 @@ export class WebhookQuoter implements Quoter { const enabledEndpoints: WebhookConfiguration[] = []; endpoints.forEach((e) => { if ( - this.ALLOW_LIST.has(e.hash) || (fillerToConfigMap.has(e.hash) && fillerToConfigMap.get(e.hash)?.enabled) || !fillerToConfigMap.has(e.hash) // default to allowing fillers not in the config ) { @@ -119,8 +124,7 @@ export class WebhookQuoter implements Quoter { timeoutSettingMs: axiosConfig.timeout, }; - try { - + try { const [hookResponse, opposite] = await Promise.all([ axios.post(endpoint, cleanRequest, axiosConfig), axios.post(endpoint, opposingCleanRequest, axiosConfig), @@ -157,14 +161,13 @@ export class WebhookQuoter implements Quoter { }, `Webhook elected not to quote: ${endpoint}` ); - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, ...rawResponse, responseType: WebhookResponseType.NON_QUOTE, - } - )); + }) + ); return null; } @@ -180,15 +183,14 @@ export class WebhookQuoter implements Quoter { }, `Webhook Response failed validation. Webhook: ${endpoint}.` ); - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, ...rawResponse, responseType: WebhookResponseType.VALIDATION_ERROR, validationError: validation.error?.details, - } - )); + }) + ); return null; } @@ -202,15 +204,14 @@ export class WebhookQuoter implements Quoter { }, `Webhook ResponseId does not match request` ); - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, ...rawResponse, responseType: WebhookResponseType.REQUEST_ID_MISMATCH, mismatchedRequestId: response.requestId, - } - )); + }) + ); return null; } @@ -227,14 +228,13 @@ export class WebhookQuoter implements Quoter { request.requestId } for endpoint ${endpoint} successful quote: ${request.amount.toString()} -> ${quote.toString()}}` ); - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, ...rawResponse, responseType: WebhookResponseType.OK, - } - )); + }) + ); //if valid quote, log the opposing side as well const opposingRequest = request.toOpposingRequest(); @@ -263,28 +263,27 @@ export class WebhookQuoter implements Quoter { { endpoint, status: e.response?.status?.toString() }, `Axios error fetching quote from ${endpoint}: ${e}` ); - const axiosResponseType = e.code === 'ECONNABORTED' ? WebhookResponseType.TIMEOUT : WebhookResponseType.HTTP_ERROR; - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + const axiosResponseType = + e.code === 'ECONNABORTED' ? WebhookResponseType.TIMEOUT : WebhookResponseType.HTTP_ERROR; + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, status: e.response?.status, data: e.response?.data, ...errorLatency, responseType: axiosResponseType, - } - )); + }) + ); } else { this.log.error({ endpoint }, `Error fetching quote from ${endpoint}: ${e}`); - this.firehose.sendAnalyticsEvent(new AnalyticsEvent( - AnalyticsEventType.WEBHOOK_RESPONSE, - { + this.firehose.sendAnalyticsEvent( + new AnalyticsEvent(AnalyticsEventType.WEBHOOK_RESPONSE, { ...requestContext, ...errorLatency, responseType: WebhookResponseType.OTHER_ERROR, otherError: `${e}`, - } - )); + }) + ); } return null; } diff --git a/lib/repositories/base.ts b/lib/repositories/base.ts index eeed8c3e..982208fd 100644 --- a/lib/repositories/base.ts +++ b/lib/repositories/base.ts @@ -28,6 +28,17 @@ export enum TimestampThreshold { TWO_MONTHS = "'2 MONTHS'", } +export type TimestampRepoRow = { + hash: string; + lastPostTimestamp: number; + blockUntilTimestamp: number; +}; + +export type DynamoTimestampRepoRow = Exclude & { + lastPostTimestamp: string; + blockUntilTimestamp: string; +}; + export abstract class BaseRedshiftRepository { constructor(readonly client: RedshiftDataClient, private readonly configs: SharedConfigs) {} @@ -67,5 +78,7 @@ export interface BaseSwitchRepository { } export interface BaseTimestampRepository { - updateTimestamp(hash: string, ts: number): Promise; + updateTimestampsBatch(toUpdate: [string, number][], ts: number): Promise; + getFillerTimestamps(hash: string): Promise; + getTimestampsBatch(hashes: string[]): Promise; } diff --git a/lib/repositories/fades-repository.ts b/lib/repositories/fades-repository.ts index 84987b64..e5528e80 100644 --- a/lib/repositories/fades-repository.ts +++ b/lib/repositories/fades-repository.ts @@ -5,8 +5,8 @@ import { BaseRedshiftRepository, SharedConfigs } from './base'; export type FadesRowType = { fillerAddress: string; - totalQuotes: number; - fadedQuotes: number; + faded: number; + postTimestamp: number; }; export class FadesRepository extends BaseRedshiftRepository { @@ -33,9 +33,9 @@ export class FadesRepository extends BaseRedshiftRepository { const stmtId = await this.executeStatement(FADE_RATE_SQL, FadesRepository.log, { waitTimeMs: 2_000 }); const response = await this.client.send(new GetStatementResultCommand({ Id: stmtId })); /* result should be in the following format - | rfqFiller | fade_rate | - |---- foo ------|---- 0.05 ------| - |---- bar ------|---- 0.01 ------| + | rfqFiller | faded | postTimestamp | + |---- foo ------|---- 1 ----|---- 12345678 ----| + |---- bar ------|---- 0 ----|---- 12222222 ----| */ const result = response.Records; if (!result) { @@ -45,8 +45,8 @@ export class FadesRepository extends BaseRedshiftRepository { const formattedResult = result.map((row) => { const formattedRow: FadesRowType = { fillerAddress: row[0].stringValue as string, - totalQuotes: Number(row[1].longValue as number), - fadedQuotes: Number(row[2].longValue as number), + faded: Number(row[1].longValue as number), + postTimestamp: Number(row[2].longValue as number), }; return formattedRow; }); @@ -62,7 +62,7 @@ WITH latestOrders AS ( SELECT * FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY filler ORDER BY createdat DESC) AS row_num FROM postedorders ) - WHERE row_num <= 30 + WHERE row_num <= 10 AND deadline < EXTRACT(EPOCH FROM GETDATE()) -- exclude orders that can still be filled ) SELECT @@ -87,16 +87,13 @@ const FADE_RATE_SQL = ` WITH ORDERS_CTE AS ( SELECT rfqFiller, - COUNT(*) AS totalQuotes, - SUM(CASE WHEN (decayStartTime < fillTimestamp) THEN 1 ELSE 0 END) AS fadedQuotes + postTimestamp, + CASE WHEN (decayStartTime < fillTimestamp) THEN 1 ELSE 0 END AS faded, FROM rfqOrdersTimestamp - GROUP BY rfqFiller ) SELECT rfqFiller, - totalQuotes, - fadedQuotes, - fadedQuotes / totalQuotes as fadeRate + postTimestamp, + faded FROM ORDERS_CTE -WHERE totalQuotes >= 10; `; diff --git a/lib/repositories/timestamp-repository.ts b/lib/repositories/timestamp-repository.ts index 142a5b41..6ff7eb06 100644 --- a/lib/repositories/timestamp-repository.ts +++ b/lib/repositories/timestamp-repository.ts @@ -3,7 +3,11 @@ import Logger from 'bunyan'; import { Entity, Table } from 'dynamodb-toolbox'; import { DYNAMO_TABLE_KEY, DYNAMO_TABLE_NAME } from '../constants'; -import { BaseTimestampRepository } from './base'; +import { BaseTimestampRepository, DynamoTimestampRepoRow, TimestampRepoRow } from './base'; + +export type BatchGetResponse = { + tableName: string; +}; export class TimestampRepository implements BaseTimestampRepository { static log: Logger; @@ -14,6 +18,8 @@ export class TimestampRepository implements BaseTimestampRepository { name: 'DynamoTimestampRepository', serializers: Logger.stdSerializers, }); + delete this.log.fields.pid; + delete this.log.fields.hostname; const table = new Table({ name: DYNAMO_TABLE_NAME.TIMESTAMP, @@ -22,10 +28,11 @@ export class TimestampRepository implements BaseTimestampRepository { }); const entity = new Entity({ - name: 'SynthSwitchEntity', + name: 'FillerTimestampEntity', attributes: { - [TimestampRepository.PARTITION_KEY]: { partitionKey: true }, - [`${DYNAMO_TABLE_KEY.TIMESTAMP}`]: { type: 'number' }, + [TimestampRepository.PARTITION_KEY]: { partitionKey: true, type: 'string' }, + [`${DYNAMO_TABLE_KEY.LAST_POST_TIMESTAMP}`]: { type: 'string' }, + [`${DYNAMO_TABLE_KEY.BLOCK_UNTIL_TIMESTAMP}`]: { type: 'string' }, }, table: table, autoExecute: true, @@ -36,20 +43,58 @@ export class TimestampRepository implements BaseTimestampRepository { private constructor( // eslint-disable-next-line - // @ts-expect-error - private readonly _switchTable: Table<'Timestamp', 'hash', null>, + private readonly table: Table<'Timestamp', 'hash', null>, private readonly entity: Entity ) {} - public async updateTimestamp(hash: string, ts: number): Promise { - await this.entity.put( + public async updateTimestampsBatch(toUpdate: [string, number][], ts: number): Promise { + await this.table.batchWrite( + toUpdate.map(([hash, postTs]) => { + return this.entity.putBatch({ + [TimestampRepository.PARTITION_KEY]: hash, + [`${DYNAMO_TABLE_KEY.LAST_POST_TIMESTAMP}`]: postTs, + [`${DYNAMO_TABLE_KEY.BLOCK_UNTIL_TIMESTAMP}`]: ts, + }); + }), { - [TimestampRepository.PARTITION_KEY]: hash, - [`${DYNAMO_TABLE_KEY.TIMESTAMP}`]: ts, - }, + execute: true, + } + ); + } + + public async getFillerTimestamps(hash: string): Promise { + const { Item } = await this.entity.get( + { hash: hash }, { execute: true, } ); + TimestampRepository.log.info({ Item }, 'get result'); + return { + hash: Item?.hash, + lastPostTimestamp: parseInt(Item?.lastPostTimestamp), + blockUntilTimestamp: parseInt(Item?.blockUntilTimestamp), + }; + } + + public async getTimestampsBatch(hashes: string[]): Promise { + const { Responses: items } = await this.table.batchGet( + hashes.map((hash) => { + return this.entity.getBatch({ + [TimestampRepository.PARTITION_KEY]: hash, + }); + }), + { + execute: true, + parse: true, + } + ); + return items[DYNAMO_TABLE_NAME.TIMESTAMP].map((row: DynamoTimestampRepoRow) => { + return { + hash: row.hash, + lastPostTimestamp: parseInt(row.lastPostTimestamp), + blockUntilTimestamp: parseInt(row.blockUntilTimestamp), + }; + }); } } diff --git a/test/repositories/shared.ts b/test/repositories/shared.ts new file mode 100644 index 00000000..3d736119 --- /dev/null +++ b/test/repositories/shared.ts @@ -0,0 +1,10 @@ +import { DynamoDBClientConfig } from '@aws-sdk/client-dynamodb'; + +export const DYNAMO_CONFIG: DynamoDBClientConfig = { + endpoint: 'http://localhost:8000', + region: 'local', + credentials: { + accessKeyId: 'fakeMyKeyId', + secretAccessKey: 'fakeSecretAccessKey', + }, +}; diff --git a/test/repositories/switch-repository.test.ts b/test/repositories/switch-repository.test.ts index b0882a6f..b53034aa 100644 --- a/test/repositories/switch-repository.test.ts +++ b/test/repositories/switch-repository.test.ts @@ -1,19 +1,11 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import { DynamoDBClient, DynamoDBClientConfig } from '@aws-sdk/client-dynamodb'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { SynthSwitchQueryParams } from '../../lib/handlers/synth-switch'; import { SwitchRepository } from '../../lib/repositories/switch-repository'; - -const dynamoConfig: DynamoDBClientConfig = { - endpoint: 'http://localhost:8000', - region: 'local', - credentials: { - accessKeyId: 'fakeMyKeyId', - secretAccessKey: 'fakeSecretAccessKey', - }, -}; +import { DYNAMO_CONFIG } from './shared'; const SWITCH: SynthSwitchQueryParams = { tokenIn: 'USDC', @@ -33,7 +25,7 @@ const NONEXISTENT_SWITCH: SynthSwitchQueryParams = { type: 'EXACT_OUTPUT', }; -const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient(dynamoConfig), { +const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient(DYNAMO_CONFIG), { marshallOptions: { convertEmptyValues: true, }, @@ -62,7 +54,7 @@ describe('put switch tests', () => { const enabled = await switchRepository.syntheticQuoteForTradeEnabled(SWITCH); expect(enabled).toBe(false); - }) + }); it('should return false for non-existent switch', async () => { await expect(switchRepository.syntheticQuoteForTradeEnabled(NONEXISTENT_SWITCH)).resolves.toBe(false); @@ -72,12 +64,12 @@ describe('put switch tests', () => { describe('static helper function tests', () => { it('correctly serializes key from trade', () => { expect(SwitchRepository.getKey(SWITCH)).toBe('usdc#1#uni#1#EXACT_INPUT'); - }) + }); it('should throw error for invalid key on parse', () => { expect(() => { // missing type SwitchRepository.parseKey('token0#1#token1#1'); }).toThrowError('Invalid key: token0#1#token1#1'); - }) -}) + }); +}); diff --git a/test/repositories/timestamp-repository.test.ts b/test/repositories/timestamp-repository.test.ts new file mode 100644 index 00000000..ed3c339b --- /dev/null +++ b/test/repositories/timestamp-repository.test.ts @@ -0,0 +1,54 @@ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { TimestampRepository } from '../../lib/repositories/timestamp-repository'; +import { DYNAMO_CONFIG } from './shared'; + +const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient(DYNAMO_CONFIG), { + marshallOptions: { + convertEmptyValues: true, + }, + unmarshallOptions: { + wrapNumbers: true, + }, +}); + +const repo = TimestampRepository.create(documentClient); + +describe('Dynamo TimestampRepo tests', () => { + it('should batch put timestamps', async () => { + const toUpdate: [string, number][] = [ + ['0x1', 1], + ['0x2', 2], + ['0x3', 3], + ]; + await expect(repo.updateTimestampsBatch(toUpdate, 4)).resolves.not.toThrow(); + const row = await repo.getFillerTimestamps('0x1'); + expect(row).toBeDefined(); + expect(row?.lastPostTimestamp).toBe(1); + }); + + it('should batch get timestamps', async () => { + const res = await repo.getTimestampsBatch(['0x1', '0x2', '0x3']); + expect(res.length).toBe(3); + expect(res).toEqual( + expect.arrayContaining([ + { + hash: '0x1', + lastPostTimestamp: 1, + blockUntilTimestamp: 4, + }, + { + hash: '0x2', + lastPostTimestamp: 2, + blockUntilTimestamp: 4, + }, + { + hash: '0x3', + lastPostTimestamp: 3, + blockUntilTimestamp: 4, + }, + ]) + ); + }); +}); From 92e6f9d5d5e646b5f6c458f264e82d49d879d8a5 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 3 Jan 2024 12:58:45 -0500 Subject: [PATCH 03/16] finish cron and db --- lib/cron/fade-rate.ts | 97 +++++++++++-------- lib/repositories/base.ts | 3 +- lib/repositories/fades-repository.ts | 28 +++--- lib/repositories/timestamp-repository.ts | 20 +++- test/crons/fade-rate.test.ts | 68 ++++++++++--- .../repositories/timestamp-repository.test.ts | 27 ++++-- 6 files changed, 167 insertions(+), 76 deletions(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 631fc083..91676212 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -5,22 +5,19 @@ import { ScheduledHandler } from 'aws-lambda/trigger/cloudwatch-events'; import { EventBridgeEvent } from 'aws-lambda/trigger/eventbridge'; import Logger from 'bunyan'; -import { - BETA_S3_KEY, - FADE_RATE_BUCKET, - FADE_RATE_S3_KEY, - FILL_RATE_THRESHOLD, - PRODUCTION_S3_KEY, - WEBHOOK_CONFIG_BUCKET, -} from '../constants'; +import { BETA_S3_KEY, PRODUCTION_S3_KEY, WEBHOOK_CONFIG_BUCKET } from '../constants'; import { CircuitBreakerMetricDimension } from '../entities'; import { checkDefined } from '../preconditions/preconditions'; import { S3WebhookConfigurationProvider } from '../providers'; -import { S3CircuitBreakerConfigurationProvider } from '../providers/circuit-breaker/s3'; -import { BaseTimestampRepository, FadesRepository, FadesRowType, SharedConfigs } from '../repositories'; +import { FadesRepository, FadesRowType, SharedConfigs, TimestampRepoRow } from '../repositories'; import { TimestampRepository } from '../repositories/timestamp-repository'; import { STAGE } from '../util/stage'; +export type FillerFades = Record; +export type FillerTimestamps = Map>; + +export const BLOCK_PER_FADE_SECS = 60 * 5; // 5 minutes + export const handler: ScheduledHandler = metricScope((metrics) => async (_event: EventBridgeEvent) => { await main(metrics); }); @@ -56,51 +53,75 @@ async function main(metrics: MetricsLogger) { const result = await fadesRepository.getFades(); if (result) { + const fillerHashes = webhookProvider.fillers(); + const fillerTimestamps = await timestampDB.getFillerTimestampsMap(fillerHashes); const addressToFillerHash = await webhookProvider.addressToFillerHash(); - const fillersNewFades = calculateFillerFadeRates(result, addressToFillerHash, log); - log.info({ fadeRates: [...fillerFadeRate.entries()] }, 'filler fade rate'); - const toUpdate = [...fillerFadeRate.entries()].filter(([, rate]) => rate >= FILL_RATE_THRESHOLD); - log.info({ toUpdate }, 'filler for which to update timestamp'); - await timestampDB.updateTimestampsBatch(toUpdate, Math.floor(Date.now() / 1000)); + // get fillers new fades from last checked timestamp: + // | rfqFiller | faded | postTimestamp | + // |---- foo ------|---- 3 ----|---- 12345678 ----| + // |---- bar ------|---- 1 ----|---- 12222222 ----| + const fillersNewFades = getFillersNewFades(result, addressToFillerHash, fillerTimestamps, log); + + const updatedTimestamps = calculateNewTimestamps( + fillerTimestamps, + fillersNewFades, + Math.floor(Date.now() / 1000), + log + ); + log.info({ updatedTimestamps }, 'filler for which to update timestamp'); + await timestampDB.updateTimestampsBatch(updatedTimestamps); } } -export function getFillersNewFades( - rows: FadesRowType[], - addressToFillerHash: Map, +/* compute blockUntil timestamp for each filler + blockedUntilTimestamp > current timestamp: skip + lastPostTimestamp < blockedUntilTimestamp < current timestamp: block for # * unit block time from now +*/ +export function calculateNewTimestamps( + fillerTimestamps: FillerTimestamps, + fillersNewFades: FillerFades, + newPostTimestamp: number, log?: Logger -): Map { - const; +): [string, number, number][] { + const updatedTimestamps: [string, number, number][] = []; + fillerTimestamps.forEach((row, hash) => { + if (row.blockUntilTimestamp > newPostTimestamp) { + return; + } + const newFades = fillersNewFades[hash]; + if (newFades) { + const blockUntilTimestamp = Math.floor(Date.now() / 1000) + newFades * BLOCK_PER_FADE_SECS; + updatedTimestamps.push([hash, newPostTimestamp, blockUntilTimestamp]); + } + }); + log?.info({ updatedTimestamps }, 'updated timestamps'); + return updatedTimestamps; } -// aggregates potentially multiple filler addresses into filler name -// and calculates fade rate for each -export function calculateFillerFadeRates( +export function getFillersNewFades( rows: FadesRowType[], addressToFillerHash: Map, + fillerTimestamps: FillerTimestamps, log?: Logger -): Map { - const fadeRateMap = new Map(); - const fillerToQuotesMap = new Map(); +): FillerFades { + const newFadesMap: FillerFades = {}; // filler hash -> # of new fades rows.forEach((row) => { const fillerAddr = row.fillerAddress.toLowerCase(); const fillerHash = addressToFillerHash.get(fillerAddr); if (!fillerHash) { - log?.info({ addressToFillerHash, fillerAddress: fillerAddr }, 'filler address not found in webhook config'); - } else { - if (!fillerToQuotesMap.has(fillerHash)) { - fillerToQuotesMap.set(fillerHash, [row.fadedQuotes, row.totalQuotes]); + log?.info({ fillerAddr }, 'filler address not found in webhook config'); + } else if ( + fillerTimestamps.has(fillerHash) && + row.postTimestamp > fillerTimestamps.get(fillerHash)!.lastPostTimestamp + ) { + if (!newFadesMap[fillerHash]) { + newFadesMap[fillerHash] = row.faded; } else { - const [fadedQuotes, totalQuotes] = fillerToQuotesMap.get(fillerHash) as [number, number]; - fillerToQuotesMap.set(fillerHash, [fadedQuotes + row.fadedQuotes, totalQuotes + row.totalQuotes]); + newFadesMap[fillerHash] += row.faded; } } }); - - fillerToQuotesMap.forEach((value, key) => { - const fadeRate = value[0] / value[1]; - fadeRateMap.set(key, fadeRate); - }); - return fadeRateMap; + log?.info({ newFadesMap }, '# of new fades by filler'); + return newFadesMap; } diff --git a/lib/repositories/base.ts b/lib/repositories/base.ts index 982208fd..e63714c4 100644 --- a/lib/repositories/base.ts +++ b/lib/repositories/base.ts @@ -78,7 +78,8 @@ export interface BaseSwitchRepository { } export interface BaseTimestampRepository { - updateTimestampsBatch(toUpdate: [string, number][], ts: number): Promise; + updateTimestampsBatch(toUpdate: [string, number, number?][]): Promise; getFillerTimestamps(hash: string): Promise; + getFillerTimestampsMap(hashes: string[]): Promise>>; getTimestampsBatch(hashes: string[]): Promise; } diff --git a/lib/repositories/fades-repository.ts b/lib/repositories/fades-repository.ts index e5528e80..b0ee00d4 100644 --- a/lib/repositories/fades-repository.ts +++ b/lib/repositories/fades-repository.ts @@ -29,13 +29,15 @@ export class FadesRepository extends BaseRedshiftRepository { await this.executeStatement(CREATE_VIEW_SQL, FadesRepository.log, { waitTimeMs: 2_000 }); } + //get latest 20 orders for each filler address, and whether they are faded or not async getFades(): Promise { const stmtId = await this.executeStatement(FADE_RATE_SQL, FadesRepository.log, { waitTimeMs: 2_000 }); const response = await this.client.send(new GetStatementResultCommand({ Id: stmtId })); /* result should be in the following format - | rfqFiller | faded | postTimestamp | - |---- foo ------|---- 1 ----|---- 12345678 ----| + | rfqFiller | faded | postTimestamp | |---- bar ------|---- 0 ----|---- 12222222 ----| + |---- foo ------|---- 1 ----|---- 12345679 ----| + |---- foo ------|---- 0 ----|---- 12345678 ----| */ const result = response.Records; if (!result) { @@ -56,14 +58,15 @@ export class FadesRepository extends BaseRedshiftRepository { } const CREATE_VIEW_SQL = ` -CREATE OR REPLACE VIEW rfqOrdersTimestamp +CREATE OR REPLACE VIEW latestRfqs AS ( WITH latestOrders AS ( SELECT * FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY filler ORDER BY createdat DESC) AS row_num FROM postedorders ) - WHERE row_num <= 10 + WHERE row_num <= 20 AND deadline < EXTRACT(EPOCH FROM GETDATE()) -- exclude orders that can still be filled + LIMIT 1000 ) SELECT latestOrders.chainid as chainId, latestOrders.filler as rfqFiller, latestOrders.startTime as decayStartTime, latestOrders.quoteid, archivedorders.filler as actualFiller, latestOrders.createdat as postTimestamp, archivedorders.txhash as txHash, archivedOrders.fillTimestamp as fillTimestamp, @@ -80,20 +83,17 @@ AND rfqFiller != '0x0000000000000000000000000000000000000000' AND chainId NOT IN (5,8001,420,421613) -- exclude mainnet goerli, polygon goerli, optimism goerli and arbitrum goerli testnets AND postTimestamp >= extract(epoch from (GETDATE() - INTERVAL '168 HOURS')) -- 7 days rolling window -); +) +ORDER BY rfqFiller, postTimestamp DESC +LIMIT 1000 `; const FADE_RATE_SQL = ` -WITH ORDERS_CTE AS ( - SELECT - rfqFiller, - postTimestamp, - CASE WHEN (decayStartTime < fillTimestamp) THEN 1 ELSE 0 END AS faded, - FROM rfqOrdersTimestamp -) SELECT rfqFiller, postTimestamp, - faded -FROM ORDERS_CTE + CASE WHEN (decayStartTime < fillTimestamp) THEN 1 ELSE 0 END AS faded +FROM latestRfqs +ORDER BY rfqFiller, postTimestamp DESC +LIMIT 1000 `; diff --git a/lib/repositories/timestamp-repository.ts b/lib/repositories/timestamp-repository.ts index 6ff7eb06..62b991ea 100644 --- a/lib/repositories/timestamp-repository.ts +++ b/lib/repositories/timestamp-repository.ts @@ -47,13 +47,13 @@ export class TimestampRepository implements BaseTimestampRepository { private readonly entity: Entity ) {} - public async updateTimestampsBatch(toUpdate: [string, number][], ts: number): Promise { + public async updateTimestampsBatch(updatedTimestamps: [string, number, number?][]): Promise { await this.table.batchWrite( - toUpdate.map(([hash, postTs]) => { + updatedTimestamps.map(([hash, lastTs, postTs]) => { return this.entity.putBatch({ [TimestampRepository.PARTITION_KEY]: hash, - [`${DYNAMO_TABLE_KEY.LAST_POST_TIMESTAMP}`]: postTs, - [`${DYNAMO_TABLE_KEY.BLOCK_UNTIL_TIMESTAMP}`]: ts, + [`${DYNAMO_TABLE_KEY.LAST_POST_TIMESTAMP}`]: lastTs, + [`${DYNAMO_TABLE_KEY.BLOCK_UNTIL_TIMESTAMP}`]: postTs, }); }), { @@ -97,4 +97,16 @@ export class TimestampRepository implements BaseTimestampRepository { }; }); } + + public async getFillerTimestampsMap(hashes: string[]): Promise>> { + const rows = await this.getTimestampsBatch(hashes); + const res = new Map>(); + rows.forEach((row) => { + res.set(row.hash, { + lastPostTimestamp: row.lastPostTimestamp, + blockUntilTimestamp: row.blockUntilTimestamp, + }); + }); + return res; + } } diff --git a/test/crons/fade-rate.test.ts b/test/crons/fade-rate.test.ts index 730fd135..bbc75c09 100644 --- a/test/crons/fade-rate.test.ts +++ b/test/crons/fade-rate.test.ts @@ -1,18 +1,40 @@ import Logger from 'bunyan'; -import { calculateFillerFadeRates } from '../../lib/cron/fade-rate'; +import { + BLOCK_PER_FADE_SECS, + calculateNewTimestamps, + FillerFades, + FillerTimestamps, + getFillersNewFades, +} from '../../lib/cron/fade-rate'; import { FadesRowType } from '../../lib/repositories'; +const now = Math.floor(Date.now() / 1000); + const FADES_ROWS: FadesRowType[] = [ - { fillerAddress: '0x1', totalQuotes: 50, fadedQuotes: 10 }, - { fillerAddress: '0x2', totalQuotes: 50, fadedQuotes: 20 }, - { fillerAddress: '0x3', totalQuotes: 100, fadedQuotes: 5 }, + // filler1 + { fillerAddress: '0x1', faded: 1, postTimestamp: now - 100 }, + { fillerAddress: '0x1', faded: 0, postTimestamp: now - 90 }, + { fillerAddress: '0x1', faded: 1, postTimestamp: now - 80 }, + { fillerAddress: '0x2', faded: 1, postTimestamp: now - 80 }, + // filler2 + { fillerAddress: '0x3', faded: 1, postTimestamp: now - 70 }, + { fillerAddress: '0x3', faded: 1, postTimestamp: now - 100 }, + // filler3 + { fillerAddress: '0x4', faded: 1, postTimestamp: now - 100 }, ]; const ADDRESS_TO_FILLER = new Map([ ['0x1', 'filler1'], ['0x2', 'filler1'], ['0x3', 'filler2'], + ['0x4', 'filler3'], +]); + +const FILLER_TIMESTAMPS: FillerTimestamps = new Map([ + ['filler1', { lastPostTimestamp: now - 150, blockUntilTimestamp: NaN }], + ['filler2', { lastPostTimestamp: now - 75, blockUntilTimestamp: now - 50 }], + ['filler3', { lastPostTimestamp: now - 101, blockUntilTimestamp: now + 1000 }], ]); // silent logger in tests @@ -20,14 +42,38 @@ const logger = Logger.createLogger({ name: 'test' }); logger.level(Logger.FATAL); describe('FadeRateCron test', () => { - describe('calculateFillerFadeRates', () => { + let newFades: FillerFades; + beforeAll(() => { + newFades = getFillersNewFades(FADES_ROWS, ADDRESS_TO_FILLER, FILLER_TIMESTAMPS, logger); + }); + + describe('getFillersNewFades', () => { it('takes into account multiple filler addresses of the same filler', () => { - expect(calculateFillerFadeRates(FADES_ROWS, ADDRESS_TO_FILLER, logger)).toEqual( - new Map([ - ['filler1', 0.3], - ['filler2', 0.05], - ]) - ); + expect(newFades).toEqual({ + filler1: 3, // count all fades in FADES_ROWS + filler2: 1, // only count postTimestamp == now - 70 + filler3: 1, + }); + }); + }); + + describe('calculateNewTimestamps', () => { + let newTimestamps: [string, number, number?][]; + + beforeAll(() => { + newTimestamps = calculateNewTimestamps(FILLER_TIMESTAMPS, newFades, now, logger); + }); + + it('calculates blockUntilTimestamp for each filler', () => { + expect(newTimestamps).toMatchObject([ + ['filler1', now, now + BLOCK_PER_FADE_SECS * 3], + ['filler2', now, now + BLOCK_PER_FADE_SECS * 1], + ]); + }); + + it('ignores fillers with blockUntilTimestamp > current timestamp', () => { + expect(newTimestamps).toHaveLength(2); + expect(newTimestamps).not.toMatchObject([['filler3', expect.anything(), expect.anything()]]); }); }); }); diff --git a/test/repositories/timestamp-repository.test.ts b/test/repositories/timestamp-repository.test.ts index ed3c339b..ef4fe06a 100644 --- a/test/repositories/timestamp-repository.test.ts +++ b/test/repositories/timestamp-repository.test.ts @@ -17,15 +17,26 @@ const repo = TimestampRepository.create(documentClient); describe('Dynamo TimestampRepo tests', () => { it('should batch put timestamps', async () => { - const toUpdate: [string, number][] = [ + const toUpdate: [string, number, number?][] = [ ['0x1', 1], - ['0x2', 2], - ['0x3', 3], + ['0x2', 2, 5], + ['0x3', 3, 6], ]; - await expect(repo.updateTimestampsBatch(toUpdate, 4)).resolves.not.toThrow(); - const row = await repo.getFillerTimestamps('0x1'); + await expect(repo.updateTimestampsBatch(toUpdate)).resolves.not.toThrow(); + let row = await repo.getFillerTimestamps('0x1'); expect(row).toBeDefined(); expect(row?.lastPostTimestamp).toBe(1); + expect(row?.blockUntilTimestamp).toBe(NaN); + + row = await repo.getFillerTimestamps('0x2'); + expect(row).toBeDefined(); + expect(row?.lastPostTimestamp).toBe(2); + expect(row?.blockUntilTimestamp).toBe(5); + + row = await repo.getFillerTimestamps('0x3'); + expect(row).toBeDefined(); + expect(row?.lastPostTimestamp).toBe(3); + expect(row?.blockUntilTimestamp).toBe(6); }); it('should batch get timestamps', async () => { @@ -36,17 +47,17 @@ describe('Dynamo TimestampRepo tests', () => { { hash: '0x1', lastPostTimestamp: 1, - blockUntilTimestamp: 4, + blockUntilTimestamp: NaN, }, { hash: '0x2', lastPostTimestamp: 2, - blockUntilTimestamp: 4, + blockUntilTimestamp: 5, }, { hash: '0x3', lastPostTimestamp: 3, - blockUntilTimestamp: 4, + blockUntilTimestamp: 6, }, ]) ); From 6f3740b769a669f65eb5985168147a9db178b9dc Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 3 Jan 2024 13:35:16 -0500 Subject: [PATCH 04/16] add unit tests --- lib/cron/fade-rate.ts | 2 +- test/crons/fade-rate.test.ts | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 91676212..cd148754 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -91,7 +91,7 @@ export function calculateNewTimestamps( } const newFades = fillersNewFades[hash]; if (newFades) { - const blockUntilTimestamp = Math.floor(Date.now() / 1000) + newFades * BLOCK_PER_FADE_SECS; + const blockUntilTimestamp = newPostTimestamp + newFades * BLOCK_PER_FADE_SECS; updatedTimestamps.push([hash, newPostTimestamp, blockUntilTimestamp]); } }); diff --git a/test/crons/fade-rate.test.ts b/test/crons/fade-rate.test.ts index bbc75c09..98af308f 100644 --- a/test/crons/fade-rate.test.ts +++ b/test/crons/fade-rate.test.ts @@ -22,6 +22,10 @@ const FADES_ROWS: FadesRowType[] = [ { fillerAddress: '0x3', faded: 1, postTimestamp: now - 100 }, // filler3 { fillerAddress: '0x4', faded: 1, postTimestamp: now - 100 }, + // filler4 + { fillerAddress: '0x5', faded: 0, postTimestamp: now - 100 }, + // filler5 + { fillerAddress: '0x6', faded: 0, postTimestamp: now - 100 }, ]; const ADDRESS_TO_FILLER = new Map([ @@ -29,12 +33,16 @@ const ADDRESS_TO_FILLER = new Map([ ['0x2', 'filler1'], ['0x3', 'filler2'], ['0x4', 'filler3'], + ['0x5', 'filler4'], + ['0x6', 'filler5'], ]); const FILLER_TIMESTAMPS: FillerTimestamps = new Map([ ['filler1', { lastPostTimestamp: now - 150, blockUntilTimestamp: NaN }], ['filler2', { lastPostTimestamp: now - 75, blockUntilTimestamp: now - 50 }], ['filler3', { lastPostTimestamp: now - 101, blockUntilTimestamp: now + 1000 }], + ['filler4', { lastPostTimestamp: now - 150, blockUntilTimestamp: NaN }], + ['filler5', { lastPostTimestamp: now - 150, blockUntilTimestamp: now + 100 }], ]); // silent logger in tests @@ -53,6 +61,8 @@ describe('FadeRateCron test', () => { filler1: 3, // count all fades in FADES_ROWS filler2: 1, // only count postTimestamp == now - 70 filler3: 1, + filler4: 0, + filler5: 0, }); }); }); @@ -71,6 +81,11 @@ describe('FadeRateCron test', () => { ]); }); + it('keep old blockUntilTimestamp if no new fades', () => { + expect(newTimestamps).not.toMatchObject([['filler5', expect.anything(), expect.anything()]]); + expect(newTimestamps).not.toMatchObject([['filler4', expect.anything(), expect.anything()]]); + }); + it('ignores fillers with blockUntilTimestamp > current timestamp', () => { expect(newTimestamps).toHaveLength(2); expect(newTimestamps).not.toMatchObject([['filler3', expect.anything(), expect.anything()]]); From 1408f17c5e9fdc2152cb943ab1b029ff3d0493b4 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 3 Jan 2024 13:52:48 -0500 Subject: [PATCH 05/16] remove duplicate imports --- lib/quoters/WebhookQuoter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/quoters/WebhookQuoter.ts b/lib/quoters/WebhookQuoter.ts index 6d30df87..17dbb1a5 100644 --- a/lib/quoters/WebhookQuoter.ts +++ b/lib/quoters/WebhookQuoter.ts @@ -19,7 +19,6 @@ import { FirehoseLogger } from '../providers/analytics'; import { CircuitBreakerConfigurationProvider } from '../providers/circuit-breaker'; import { FillerComplianceConfigurationProvider } from '../providers/compliance'; import { timestampInMstoISOString } from '../util/time'; -import { Quoter, QuoterType } from '.'; // TODO: shorten, maybe take from env config const WEBHOOK_TIMEOUT_MS = 500; From ae4e70e4927c2ba87ef662a154e423205bad36a8 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 3 Jan 2024 13:57:11 -0500 Subject: [PATCH 06/16] revert changes in s3 provider --- lib/providers/circuit-breaker/s3.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/lib/providers/circuit-breaker/s3.ts b/lib/providers/circuit-breaker/s3.ts index f2ea21fe..0fb0179e 100644 --- a/lib/providers/circuit-breaker/s3.ts +++ b/lib/providers/circuit-breaker/s3.ts @@ -3,23 +3,17 @@ import { NodeHttpHandler } from '@smithy/node-http-handler'; import { MetricsLogger, Unit } from 'aws-embedded-metrics'; import Logger from 'bunyan'; -import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; -import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { CircuitBreakerConfiguration, CircuitBreakerConfigurationProvider } from '.'; import { Metric } from '../../entities'; import { checkDefined } from '../../preconditions/preconditions'; -import { BaseTimestampRepository } from '../../repositories'; -import { TimestampRepository } from '../../repositories/timestamp-repository'; export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConfigurationProvider { private log: Logger; private fillers: CircuitBreakerConfiguration[]; private lastUpdatedTimestamp: number; private client: S3Client; - private timestampDB: BaseTimestampRepository; - - // try to refetch endpoints every minute private static UPDATE_PERIOD_MS = 1 * 60000; + private static FILL_RATE_THRESHOLD = 0.75; constructor(_log: Logger, private bucket: string, private key: string) { this.log = _log.child({ quoter: 'S3CircuitBreakerConfigurationProvider' }); @@ -30,15 +24,6 @@ export class S3CircuitBreakerConfigurationProvider implements CircuitBreakerConf requestTimeout: 500, }), }); - const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient({}), { - marshallOptions: { - convertEmptyValues: true, - }, - unmarshallOptions: { - wrapNumbers: true, - }, - }); - this.timestampDB = TimestampRepository.create(documentClient); } async getConfigurations(): Promise { From 60dba7cc33224ddde4da2094029f8c0503e80a1c Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Wed, 3 Jan 2024 14:12:53 -0500 Subject: [PATCH 07/16] reset changes to webhookQuoter --- lib/quoters/WebhookQuoter.ts | 3 +++ lib/repositories/switch-repository.ts | 2 -- lib/repositories/timestamp-repository.ts | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/quoters/WebhookQuoter.ts b/lib/quoters/WebhookQuoter.ts index 17dbb1a5..f8f9bb15 100644 --- a/lib/quoters/WebhookQuoter.ts +++ b/lib/quoters/WebhookQuoter.ts @@ -27,6 +27,7 @@ const WEBHOOK_TIMEOUT_MS = 500; // endpoints must return well-formed QuoteResponse JSON export class WebhookQuoter implements Quoter { private log: Logger; + private readonly ALLOW_LIST: Set; constructor( _log: Logger, @@ -37,6 +38,7 @@ export class WebhookQuoter implements Quoter { _allow_list: Set = new Set(['1ed189c4b20479e36acf74e2bc87e03bfdce765ecba6696970caee8299fc005f']) ) { this.log = _log.child({ quoter: 'WebhookQuoter' }); + this.ALLOW_LIST = _allow_list; } public async quote(request: QuoteRequest): Promise { @@ -70,6 +72,7 @@ export class WebhookQuoter implements Quoter { const enabledEndpoints: WebhookConfiguration[] = []; endpoints.forEach((e) => { if ( + this.ALLOW_LIST.has(e.hash) || (fillerToConfigMap.has(e.hash) && fillerToConfigMap.get(e.hash)?.enabled) || !fillerToConfigMap.has(e.hash) // default to allowing fillers not in the config ) { diff --git a/lib/repositories/switch-repository.ts b/lib/repositories/switch-repository.ts index 2e9de20f..4b1ecd5e 100644 --- a/lib/repositories/switch-repository.ts +++ b/lib/repositories/switch-repository.ts @@ -61,7 +61,6 @@ export class SwitchRepository implements BaseSwitchRepository { { execute: true, consistent: true } ); - SwitchRepository.log.info({ res: result.Item }, 'get result'); if (result.Item && BigNumber.from(result.Item.lower).lte(amount)) { return result.Item.enabled; } else { @@ -71,7 +70,6 @@ export class SwitchRepository implements BaseSwitchRepository { } public async putSynthSwitch(trade: SynthSwitchTrade, lower: string, enabled: boolean): Promise { - SwitchRepository.log.info({ pk: `${SwitchRepository.getKey(trade)}` }, 'put pk'); await this.switchEntity.put( { [PARTITION_KEY]: `${SwitchRepository.getKey(trade)}`, diff --git a/lib/repositories/timestamp-repository.ts b/lib/repositories/timestamp-repository.ts index 62b991ea..3864ccd4 100644 --- a/lib/repositories/timestamp-repository.ts +++ b/lib/repositories/timestamp-repository.ts @@ -69,7 +69,6 @@ export class TimestampRepository implements BaseTimestampRepository { execute: true, } ); - TimestampRepository.log.info({ Item }, 'get result'); return { hash: Item?.hash, lastPostTimestamp: parseInt(Item?.lastPostTimestamp), From 039fbfaa306bd82611e2f994f4cfb82f5e1158cc Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Thu, 4 Jan 2024 16:21:18 -0500 Subject: [PATCH 08/16] fix new filler cold start --- lib/cron/fade-rate.ts | 16 +++++++++------- test/crons/fade-rate.test.ts | 27 ++++++++++++++++++--------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index cd148754..3a927014 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -33,6 +33,7 @@ async function main(metrics: MetricsLogger) { const stage = process.env['stage']; const s3Key = stage === STAGE.BETA ? BETA_S3_KEY : PRODUCTION_S3_KEY; const webhookProvider = new S3WebhookConfigurationProvider(log, `${WEBHOOK_CONFIG_BUCKET}-${stage}-1`, s3Key); + await webhookProvider.fetchEndpoints(); const documentClient = DynamoDBDocumentClient.from(new DynamoDBClient({}), { marshallOptions: { convertEmptyValues: true, @@ -85,13 +86,14 @@ export function calculateNewTimestamps( log?: Logger ): [string, number, number][] { const updatedTimestamps: [string, number, number][] = []; - fillerTimestamps.forEach((row, hash) => { - if (row.blockUntilTimestamp > newPostTimestamp) { + Object.entries(fillersNewFades).forEach((row) => { + const hash = row[0]; + const fades = row[1]; + if (fillerTimestamps.has(hash) && fillerTimestamps.get(hash)!.blockUntilTimestamp > newPostTimestamp) { return; } - const newFades = fillersNewFades[hash]; - if (newFades) { - const blockUntilTimestamp = newPostTimestamp + newFades * BLOCK_PER_FADE_SECS; + if (fades) { + const blockUntilTimestamp = newPostTimestamp + fades * BLOCK_PER_FADE_SECS; updatedTimestamps.push([hash, newPostTimestamp, blockUntilTimestamp]); } }); @@ -112,8 +114,8 @@ export function getFillersNewFades( if (!fillerHash) { log?.info({ fillerAddr }, 'filler address not found in webhook config'); } else if ( - fillerTimestamps.has(fillerHash) && - row.postTimestamp > fillerTimestamps.get(fillerHash)!.lastPostTimestamp + (fillerTimestamps.has(fillerHash) && row.postTimestamp > fillerTimestamps.get(fillerHash)!.lastPostTimestamp) || + !fillerTimestamps.has(fillerHash) ) { if (!newFadesMap[fillerHash]) { newFadesMap[fillerHash] = row.faded; diff --git a/test/crons/fade-rate.test.ts b/test/crons/fade-rate.test.ts index 98af308f..6198a973 100644 --- a/test/crons/fade-rate.test.ts +++ b/test/crons/fade-rate.test.ts @@ -26,6 +26,8 @@ const FADES_ROWS: FadesRowType[] = [ { fillerAddress: '0x5', faded: 0, postTimestamp: now - 100 }, // filler5 { fillerAddress: '0x6', faded: 0, postTimestamp: now - 100 }, + // filler6 + { fillerAddress: '0x7', faded: 1, postTimestamp: now - 100 }, ]; const ADDRESS_TO_FILLER = new Map([ @@ -35,6 +37,7 @@ const ADDRESS_TO_FILLER = new Map([ ['0x4', 'filler3'], ['0x5', 'filler4'], ['0x6', 'filler5'], + ['0x7', 'filler6'], ]); const FILLER_TIMESTAMPS: FillerTimestamps = new Map([ @@ -63,32 +66,38 @@ describe('FadeRateCron test', () => { filler3: 1, filler4: 0, filler5: 0, + filler6: 1, }); }); }); describe('calculateNewTimestamps', () => { - let newTimestamps: [string, number, number?][]; + let newTimestamps: [string, number, number][]; beforeAll(() => { newTimestamps = calculateNewTimestamps(FILLER_TIMESTAMPS, newFades, now, logger); }); it('calculates blockUntilTimestamp for each filler', () => { - expect(newTimestamps).toMatchObject([ - ['filler1', now, now + BLOCK_PER_FADE_SECS * 3], - ['filler2', now, now + BLOCK_PER_FADE_SECS * 1], - ]); + expect(newTimestamps).toEqual( + expect.arrayContaining([ + ['filler1', now, now + BLOCK_PER_FADE_SECS * 3], + ['filler2', now, now + BLOCK_PER_FADE_SECS * 1], + ]) + ); + }); + + it('notices new fillers not already in fillerTimestamps', () => { + expect(newTimestamps).toEqual(expect.arrayContaining([['filler6', now, now + BLOCK_PER_FADE_SECS * 1]])); }); it('keep old blockUntilTimestamp if no new fades', () => { - expect(newTimestamps).not.toMatchObject([['filler5', expect.anything(), expect.anything()]]); - expect(newTimestamps).not.toMatchObject([['filler4', expect.anything(), expect.anything()]]); + expect(newTimestamps).not.toContain([['filler5', expect.anything(), expect.anything()]]); + expect(newTimestamps).not.toContain([['filler4', expect.anything(), expect.anything()]]); }); it('ignores fillers with blockUntilTimestamp > current timestamp', () => { - expect(newTimestamps).toHaveLength(2); - expect(newTimestamps).not.toMatchObject([['filler3', expect.anything(), expect.anything()]]); + expect(newTimestamps).not.toContain([['filler3', expect.anything(), expect.anything()]]); }); }); }); From 6423925d81cb428fca1a86dfcebb456eb6aa883d Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 5 Jan 2024 13:35:01 -0500 Subject: [PATCH 09/16] update comments --- lib/cron/fade-rate.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 3a927014..1f6b6465 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -51,6 +51,13 @@ async function main(metrics: MetricsLogger) { }; const fadesRepository = FadesRepository.create(sharedConfig); await fadesRepository.createFadesView(); + /* + query redshift for recent orders + | fillerAddress | faded | postTimestamp | + |---- 0x1 ------|---- 0 ----|---- 12222222 ---| + |---- 0x2 ------|---- 1 ----|---- 12345679 --| + |---- 0x1 ------|---- 0 ----|---- 12345678 ---| + */ const result = await fadesRepository.getFades(); if (result) { @@ -58,8 +65,8 @@ async function main(metrics: MetricsLogger) { const fillerTimestamps = await timestampDB.getFillerTimestampsMap(fillerHashes); const addressToFillerHash = await webhookProvider.addressToFillerHash(); - // get fillers new fades from last checked timestamp: - // | rfqFiller | faded | postTimestamp | + // aggregated # of fades by filler entity (not address) + // | hash | faded | postTimestamp | // |---- foo ------|---- 3 ----|---- 12345678 ----| // |---- bar ------|---- 1 ----|---- 12222222 ----| const fillersNewFades = getFillersNewFades(result, addressToFillerHash, fillerTimestamps, log); From bf74930766aa1c02742089cf073b28dd34eb3087 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 5 Jan 2024 13:35:49 -0500 Subject: [PATCH 10/16] function comment --- lib/cron/fade-rate.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 1f6b6465..5e65b622 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -108,6 +108,9 @@ export function calculateNewTimestamps( return updatedTimestamps; } +/* find the number of new fades, for each filler entity, from + the last time this cron is run +*/ export function getFillersNewFades( rows: FadesRowType[], addressToFillerHash: Map, From 151825638661c1cba46b3212521e18d0cd86860f Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 5 Jan 2024 13:42:44 -0500 Subject: [PATCH 11/16] fix comment --- lib/cron/fade-rate.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 5e65b622..c7eedf0c 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -71,6 +71,9 @@ async function main(metrics: MetricsLogger) { // |---- bar ------|---- 1 ----|---- 12222222 ----| const fillersNewFades = getFillersNewFades(result, addressToFillerHash, fillerTimestamps, log); + // | hash |lastPostTimestamp|blockUntilTimestamp| + // |---- foo ----|---- 1300000 ----|---- now + fades * block_per_fade ----| + // |---- bar ----|---- 1300000 ----|---- 13500000 ----| const updatedTimestamps = calculateNewTimestamps( fillerTimestamps, fillersNewFades, From 4f543360612092d61be689e34c6d4461c4c2f9ad Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 5 Jan 2024 13:49:38 -0500 Subject: [PATCH 12/16] fix comment --- lib/cron/fade-rate.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index c7eedf0c..48cd60d8 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -66,9 +66,9 @@ async function main(metrics: MetricsLogger) { const addressToFillerHash = await webhookProvider.addressToFillerHash(); // aggregated # of fades by filler entity (not address) - // | hash | faded | postTimestamp | - // |---- foo ------|---- 3 ----|---- 12345678 ----| - // |---- bar ------|---- 1 ----|---- 12222222 ----| + // | hash | faded | + // |-- foo --|--- 3 ---|` + // |-- bar --|--- 1 ---| const fillersNewFades = getFillersNewFades(result, addressToFillerHash, fillerTimestamps, log); // | hash |lastPostTimestamp|blockUntilTimestamp| From d135b4a4b24350473edcbbd381eac5f11242a1a6 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 12 Jan 2024 10:17:13 -0500 Subject: [PATCH 13/16] rename table --- lib/constants.ts | 2 +- lib/repositories/timestamp-repository.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/constants.ts b/lib/constants.ts index 3dcc2c47..4f146d5d 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -12,7 +12,7 @@ export const BETA_COMPLIANCE_S3_KEY = 'beta.json'; export const DYNAMO_TABLE_NAME = { FADES: 'Fades', SYNTHETIC_SWITCH_TABLE: 'SyntheticSwitchTable', - TIMESTAMP: 'Timestamp', + FILLER_CB_TIMESTAMPS: 'FillerCBTimestamps', }; export const DYNAMO_TABLE_KEY = { diff --git a/lib/repositories/timestamp-repository.ts b/lib/repositories/timestamp-repository.ts index 3864ccd4..cc6dceb8 100644 --- a/lib/repositories/timestamp-repository.ts +++ b/lib/repositories/timestamp-repository.ts @@ -22,7 +22,7 @@ export class TimestampRepository implements BaseTimestampRepository { delete this.log.fields.hostname; const table = new Table({ - name: DYNAMO_TABLE_NAME.TIMESTAMP, + name: DYNAMO_TABLE_NAME.FILLER_CB_TIMESTAMPS, partitionKey: TimestampRepository.PARTITION_KEY, DocumentClient: documentClient, }); @@ -88,7 +88,7 @@ export class TimestampRepository implements BaseTimestampRepository { parse: true, } ); - return items[DYNAMO_TABLE_NAME.TIMESTAMP].map((row: DynamoTimestampRepoRow) => { + return items[DYNAMO_TABLE_NAME.FILLER_CB_TIMESTAMPS].map((row: DynamoTimestampRepoRow) => { return { hash: row.hash, lastPostTimestamp: parseInt(row.lastPostTimestamp), From eff0b590da6ff2cc5f4b0f9e9332a189e8ff4231 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 12 Jan 2024 10:21:26 -0500 Subject: [PATCH 14/16] create table in cdk --- bin/config.ts | 1 + bin/stacks/cron-stack.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/bin/config.ts b/bin/config.ts index 58804d96..b70c8edc 100644 --- a/bin/config.ts +++ b/bin/config.ts @@ -4,5 +4,6 @@ import { TableCapacityConfig } from './stacks/cron-stack'; export const PROD_TABLE_CAPACITY: TableCapacityConfig = { fadeRate: { billingMode: BillingMode.PROVISIONED, readCapacity: 50, writeCapacity: 5 }, + timestamps: { billingMode: BillingMode.PROVISIONED, readCapacity: 50, writeCapacity: 5 }, synthSwitch: { billingMode: BillingMode.PROVISIONED, readCapacity: 2000, writeCapacity: 5 }, }; diff --git a/bin/stacks/cron-stack.ts b/bin/stacks/cron-stack.ts index e59e5933..d703c3f1 100644 --- a/bin/stacks/cron-stack.ts +++ b/bin/stacks/cron-stack.ts @@ -31,6 +31,7 @@ type TableCapacityOptions = { export type TableCapacityConfig = { fadeRate: TableCapacityOptions; synthSwitch: TableCapacityOptions; + timestamps: TableCapacityOptions; }; export interface CronStackProps extends cdk.NestedStackProps { @@ -177,6 +178,19 @@ export class CronStack extends cdk.NestedStack { ...PROD_TABLE_CAPACITY.synthSwitch, }); this.alarmsPerTable(synthSwitchTable, DYNAMO_TABLE_NAME.SYNTHETIC_SWITCH_TABLE, chatbotTopic); + + const fillerCBTimestampsTable = new aws_dynamo.Table(this, `${SERVICE_NAME}FillerCBTimestampsTable`, { + tableName: DYNAMO_TABLE_NAME.FILLER_CB_TIMESTAMPS, + partitionKey: { + name: 'hash', + type: aws_dynamo.AttributeType.STRING, + }, + deletionProtection: true, + pointInTimeRecovery: true, + contributorInsightsEnabled: true, + ...PROD_TABLE_CAPACITY.timestamps, + }); + this.alarmsPerTable(fillerCBTimestampsTable, DYNAMO_TABLE_NAME.FILLER_CB_TIMESTAMPS, chatbotTopic); } private alarmsPerTable(table: aws_dynamo.Table, name: string, chatbotSNSTopic?: ITopic): void { From ba832055918773efaf21598a9de1af325150d44f Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 12 Jan 2024 10:23:03 -0500 Subject: [PATCH 15/16] rename --- lib/cron/fade-rate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cron/fade-rate.ts b/lib/cron/fade-rate.ts index 48cd60d8..5b8f09c8 100644 --- a/lib/cron/fade-rate.ts +++ b/lib/cron/fade-rate.ts @@ -66,7 +66,7 @@ async function main(metrics: MetricsLogger) { const addressToFillerHash = await webhookProvider.addressToFillerHash(); // aggregated # of fades by filler entity (not address) - // | hash | faded | + // | hash | fades | // |-- foo --|--- 3 ---|` // |-- bar --|--- 1 ---| const fillersNewFades = getFillersNewFades(result, addressToFillerHash, fillerTimestamps, log); From be33af070d1b1449119ca86649135f7d378c1d86 Mon Sep 17 00:00:00 2001 From: ConjunctiveNormalForm Date: Fri, 12 Jan 2024 10:26:34 -0500 Subject: [PATCH 16/16] update jest-dynamo config file --- jest-dynamodb-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest-dynamodb-config.js b/jest-dynamodb-config.js index d76b13c7..59d08dde 100644 --- a/jest-dynamodb-config.js +++ b/jest-dynamodb-config.js @@ -11,7 +11,7 @@ module.exports = { ProvisionedThroughput: { ReadCapacityUnits: 10, WriteCapacityUnits: 10 }, }, { - TableName: 'Timestamp', + TableName: 'FillerCBTimestamps', KeySchema: [ { AttributeName: 'hash', KeyType: 'HASH' }, ],