From 5a026f3871c424cf3fa0d1cd559720b547974e68 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 17 Jul 2023 08:45:02 +1200 Subject: [PATCH] DPLT-1044 Expose context method for writing custom Grafana metrics (#126) --- .../__snapshots__/metrics.test.js.snap | 4 ++ indexer-js-queue-handler/indexer.js | 18 ++++++-- indexer-js-queue-handler/indexer.test.js | 45 ++++++++++++++++++- indexer-js-queue-handler/metrics.js | 14 ++++-- indexer-js-queue-handler/metrics.test.js | 2 +- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap b/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap index 6886b82e0..3fa7e7bac 100644 --- a/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap @@ -18,6 +18,10 @@ exports[`Metrics writes the block height for an indexer function 1`] = ` "Name": "STAGE", "Value": "dev", }, + { + "Name": "HISTORICAL", + "Value": false, + }, ], "MetricName": "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", "Unit": "None", diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index 63e69a535..648533e09 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -46,7 +46,7 @@ export default class Indexer { functionSubsegment.addAnnotation('indexer_function', function_name); simultaneousPromises.push(this.writeLog(function_name, block_height, runningMessage)); - simultaneousPromises.push(this.deps.metrics.putBlockHeight(indexerFunction.account_id, indexerFunction.function_name, block_height)); + simultaneousPromises.push(this.deps.metrics.putBlockHeight(indexerFunction.account_id, indexerFunction.function_name, is_historical, block_height)); const hasuraRoleName = function_name.split('/')[0].replace(/[.-]/g, '_'); const functionNameWithoutAccount = function_name.split('/')[1].replace(/[.-]/g, '_'); @@ -74,7 +74,7 @@ export default class Indexer { const vm = new VM({timeout: 3000, allowAsync: true}); const mutationsReturnValue = {mutations: [], variables: {}, keysValues: {}}; const context = options.imperative - ? this.buildImperativeContextForFunction(function_name, functionNameWithoutAccount, block_height, hasuraRoleName) + ? this.buildImperativeContextForFunction(function_name, functionNameWithoutAccount, block_height, hasuraRoleName, is_historical) : this.buildFunctionalContextForFunction(mutationsReturnValue, function_name, block_height); vm.freeze(blockWithHelpers, 'block'); @@ -225,7 +225,7 @@ export default class Indexer { }; } - buildImperativeContextForFunction(functionName, functionNameWithoutAccount, block_height, hasuraRoleName) { + buildImperativeContextForFunction(functionName, functionNameWithoutAccount, block_height, hasuraRoleName, is_historical) { return { graphql: async (operation, variables) => { try { @@ -254,7 +254,17 @@ export default class Indexer { }, log: async (log) => { return await this.writeLog(functionName, block_height, log); - } + }, + putMetric: (name, value) => { + const [accountId, fnName] = functionName.split('/'); + return this.deps.metrics.putCustomMetric( + accountId, + fnName, + is_historical, + `CUSTOM_${name}`, + value + ); + }, }; } diff --git a/indexer-js-queue-handler/indexer.test.js b/indexer-js-queue-handler/indexer.test.js index ce2f39b3b..be955f973 100644 --- a/indexer-js-queue-handler/indexer.test.js +++ b/indexer-js-queue-handler/indexer.test.js @@ -821,7 +821,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(block_height, functions, false); - expect(metrics.putBlockHeight).toHaveBeenCalledWith('buildnear.testnet', 'test', block_height); + expect(metrics.putBlockHeight).toHaveBeenCalledWith('buildnear.testnet', 'test', false, block_height); }); test('does not attach the hasura admin secret header when no role specified', async () => { @@ -895,6 +895,49 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in ]); }); + test('allows writing of custom metrics', async () => { + const mockFetch = jest.fn(() => ({ + status: 200, + json: async () => ({ + errors: null, + }), + })); + const block_height = 456; + const mockS3 = { + getObject: jest.fn(() => ({ + promise: () => Promise.resolve({ + Body: { + toString: () => JSON.stringify({ + chunks: [], + header: { + height: block_height + } + }) + } + }) + })), + }; + const metrics = { + putBlockHeight: () => {}, + putCustomMetric: jest.fn(), + }; + const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, awsXray: mockAwsXray, metrics }); + + const functions = {}; + functions['buildnear.testnet/test'] = {code:` + context.putMetric('TEST_METRIC', 1) + `}; + await indexer.runFunctions(block_height, functions, true, { imperative: true }); + + expect(metrics.putCustomMetric).toHaveBeenCalledWith( + 'buildnear.testnet', + 'test', + true, + 'CUSTOM_TEST_METRIC', + 1 + ); + }); + // The unhandled promise causes problems with test reporting. // Note unhandled promise rejections fail to proceed to the next function on AWS Lambda test.skip('Indexer.runFunctions() continues despite promise rejection, unable to log rejection', async () => { diff --git a/indexer-js-queue-handler/metrics.js b/indexer-js-queue-handler/metrics.js index 52909fe30..de0882052 100644 --- a/indexer-js-queue-handler/metrics.js +++ b/indexer-js-queue-handler/metrics.js @@ -7,12 +7,16 @@ export default class Metrics { this.namespace = namespace; } - putBlockHeight(accountId, functionName, height) { + putBlockHeight(accountId, functionName, isHistorical, height) { + return this.putCustomMetric(accountId, functionName, isHistorical, "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", height); + } + + putCustomMetric(accountId, functionName, isHistorical, metricName, value) { return this.cloudwatch .putMetricData({ MetricData: [ { - MetricName: "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", + MetricName: metricName, Dimensions: [ { Name: "ACCOUNT_ID", @@ -26,9 +30,13 @@ export default class Metrics { Name: "STAGE", Value: process.env.STAGE, }, + { + Name: "HISTORICAL", + Value: isHistorical, + }, ], Unit: "None", - Value: height, + Value: value, }, ], Namespace: this.namespace, diff --git a/indexer-js-queue-handler/metrics.test.js b/indexer-js-queue-handler/metrics.test.js index aa58b8875..fc9b74aaf 100644 --- a/indexer-js-queue-handler/metrics.test.js +++ b/indexer-js-queue-handler/metrics.test.js @@ -22,7 +22,7 @@ describe('Metrics', () => { }; const metrics = new Metrics('test', cloudwatch); - await metrics.putBlockHeight('morgs.near', 'test', 2); + await metrics.putBlockHeight('morgs.near', 'test', false, 2); expect(cloudwatch.putMetricData).toBeCalledTimes(1); expect(cloudwatch.putMetricData.mock.calls[0]).toMatchSnapshot()