From 805dfbfc00a927b09413be9d71999dcd0235eecb Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:19:18 +0400 Subject: [PATCH 1/8] fix: keys api urls validation when CLI --- src/common/config/config.service.ts | 2 +- src/common/config/env.validation.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/config/config.service.ts b/src/common/config/config.service.ts index 270747b..d3a5ddf 100644 --- a/src/common/config/config.service.ts +++ b/src/common/config/config.service.ts @@ -7,7 +7,7 @@ export class ConfigService extends ConfigServiceSource { * List of env variables that should be hidden */ public get secrets(): string[] { - return [...this.get('EL_RPC_URLS'), ...this.get('CL_API_URLS'), ...this.get('KEYSAPI_API_URLS')]; + return [...this.get('EL_RPC_URLS'), ...this.get('CL_API_URLS'), ...(this.get('KEYSAPI_API_URLS') ?? [])]; } public get(key: T): EnvironmentVariables[T] { diff --git a/src/common/config/env.validation.ts b/src/common/config/env.validation.ts index d1e6705..20eead5 100644 --- a/src/common/config/env.validation.ts +++ b/src/common/config/env.validation.ts @@ -153,7 +153,8 @@ export class EnvironmentVariables { @IsArray() @ArrayMinSize(1) @Transform(({ value }) => value.split(',')) - public KEYSAPI_API_URLS!: string[]; + @ValidateIf((vars) => vars.WORKING_MODE === WorkingMode.Daemon) + public KEYSAPI_API_URLS: string[]; @IsInt() @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) From 2493fce41e0d98a694eddda46d19d17b09526f9e Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:21:56 +0400 Subject: [PATCH 2/8] fix: remove extra service from command --- src/cli/commands/prove.command.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cli/commands/prove.command.ts b/src/cli/commands/prove.command.ts index 2e50c2f..7959763 100644 --- a/src/cli/commands/prove.command.ts +++ b/src/cli/commands/prove.command.ts @@ -7,7 +7,6 @@ import { CsmContract } from '../../common/contracts/csm-contract.service'; import { ProverService } from '../../common/prover/prover.service'; import { KeyInfoFn } from '../../common/prover/types'; import { Consensus } from '../../common/providers/consensus/consensus'; -import { Execution } from '../../common/providers/execution/execution'; type ProofOptions = { nodeOperatorId: string; @@ -35,7 +34,6 @@ export class ProveCommand extends CommandRunner { protected readonly inquirerService: InquirerService, protected readonly csm: CsmContract, protected readonly consensus: Consensus, - protected readonly execution: Execution, protected readonly prover: ProverService, ) { super(); From 4625c92ff0e2312ed33651f1a1a27e0e8d3f2b1a Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:24:01 +0400 Subject: [PATCH 3/8] fix: use ora commonjs ver --- src/common/utils/download-progress/download-progress.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/utils/download-progress/download-progress.ts b/src/common/utils/download-progress/download-progress.ts index 60f527a..6479ec9 100644 --- a/src/common/utils/download-progress/download-progress.ts +++ b/src/common/utils/download-progress/download-progress.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; -import ora, { Ora } from 'ora'; +import ora = require('ora-classic'); +import { Ora } from 'ora-classic'; import { IncomingHttpHeaders } from 'undici/types/header'; import BodyReadable from 'undici/types/readable'; From c20a134bda1498245d844d55ec5e097ede5fd731 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:24:49 +0400 Subject: [PATCH 4/8] feat: retries for CL and KeysAPI --- src/common/providers/base/rest-provider.ts | 81 +++++++++++++++------ src/common/providers/base/utils/func.ts | 58 +++++++++++++++ src/common/providers/consensus/consensus.ts | 41 ++++++++--- src/common/providers/keysapi/keysapi.ts | 46 +++++++++--- 4 files changed, 182 insertions(+), 44 deletions(-) create mode 100644 src/common/providers/base/utils/func.ts diff --git a/src/common/providers/base/rest-provider.ts b/src/common/providers/base/rest-provider.ts index e6611ac..700ee6f 100644 --- a/src/common/providers/base/rest-provider.ts +++ b/src/common/providers/base/rest-provider.ts @@ -3,43 +3,80 @@ import { request } from 'undici'; import { IncomingHttpHeaders } from 'undici/types/header'; import BodyReadable from 'undici/types/readable'; -import { PrometheusService } from '../../prometheus/prometheus.service'; +import { RequestOptions, RequestPolicy, rejectDelay, retrier } from './utils/func'; +import { PrometheusService } from '../../prometheus'; -export interface RequestPolicy { - timeout: number; - maxRetries: number; - fallbacks: Array; -} +export type RetryOptions = RequestOptions & RequestPolicy & { useFallbackOnRejected?: (err: Error, current_error: Error) => boolean; useFallbackOnResolved?: (data: any) => boolean} -export interface RequestOptions { - requestPolicy?: RequestPolicy; - signal?: AbortSignal; - headers?: Record; +class RequestError extends Error { + constructor(message: string, public readonly statusCode?: number) { + super(message); + } } export abstract class BaseRestProvider { - protected readonly mainUrl: string; + protected readonly baseUrls: string[]; protected readonly requestPolicy: RequestPolicy; protected constructor( urls: Array, responseTimeout: number, maxRetries: number, + retryDelay: number, protected readonly logger: LoggerService, protected readonly prometheus?: PrometheusService, ) { - this.mainUrl = urls[0]; + this.baseUrls = urls; this.requestPolicy = { timeout: responseTimeout, - maxRetries: maxRetries, - fallbacks: urls.slice(1), + maxRetries, + retryDelay, }; } - // TODO: Request should have: - // 1. metrics (if it is daemon mode) - // 2. retries - // 3. fallbacks + protected async retryRequest( + callback: (apiURL: string, options?: RequestOptions) => Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }>, + options?: RetryOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + options = { + ...this.requestPolicy, + useFallbackOnRejected: (() => true), // use fallback on error as default + useFallbackOnResolved: (() => false), // do NOT use fallback on success as default + ...options, + }; + const retry = retrier(this.logger, options.maxRetries, this.requestPolicy.retryDelay, 10000, true); + let res; + let err = Error(''); + for (let i = 0; i < this.baseUrls.length; i++) { + if (res) break; + res = await callback(this.baseUrls[i], options) + .catch(rejectDelay(this.requestPolicy.retryDelay)) + .catch(() => retry(() => callback(this.baseUrls[i], options))) + .then((r: any) => { + if (options?.useFallbackOnResolved && options.useFallbackOnResolved(r)) { + err = Error('Unresolved data on a successful CL API response'); + return undefined; + } + return r; + }) + .catch((current_error: any) => { + if (options?.useFallbackOnRejected && options.useFallbackOnRejected(err, current_error)) { + err = current_error; + return undefined; + } + throw current_error; + }); + if (i == this.baseUrls.length - 1 && !res) { + err.message = `Error while doing CL API request on all passed URLs. ${err.message}`; + throw err; + } + if (!res) { + this.logger.warn(`${err.message}. Error while doing CL API request. Will try to switch to another API URL`); + } + } + + return res; + } protected async baseGet( base: string, @@ -58,10 +95,10 @@ export abstract class BaseRestProvider { }); if (statusCode !== 200) { const hostname = new URL(base).hostname; - throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`); + throw new RequestError(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, statusCode); } return { body: body, headers: headers }; - } + }; protected async basePost( base: string, @@ -84,8 +121,8 @@ export abstract class BaseRestProvider { }); if (statusCode !== 200) { const hostname = new URL(base).hostname; - throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`); + throw new RequestError(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, statusCode); } return { body: body, headers: headers }; - } + }; } diff --git a/src/common/providers/base/utils/func.ts b/src/common/providers/base/utils/func.ts new file mode 100644 index 0000000..80036d5 --- /dev/null +++ b/src/common/providers/base/utils/func.ts @@ -0,0 +1,58 @@ +import { LoggerService } from '@nestjs/common'; + +export interface RequestPolicy { + timeout: number; + maxRetries: number; + retryDelay: number; +} + +export interface RequestOptions { + requestPolicy?: RequestPolicy; + signal?: AbortSignal; + headers?: Record; +} + +export const rejectDelay = (delayMs: number) => (err: any) => { + return new Promise(function (resolve, reject) { + setTimeout(() => reject(err), delayMs); + }); +}; + +export const sleep = (delayMs: number) => { + return new Promise(function (resolve) { + setTimeout(() => resolve(), delayMs); + }); +}; + +export const retrier = ( + logger: LoggerService, + defaultMaxRetryCount = 3, + defaultMinBackoffMs = 1000, + defaultMaxBackoffMs = 60000, + defaultLogWarning = false, +) => { + return async ( + callback: () => Promise, + maxRetryCount?: number, + minBackoffMs?: number, + maxBackoffMs?: number, + logWarning?: boolean, + ): Promise => { + maxRetryCount = maxRetryCount ?? defaultMaxRetryCount; + minBackoffMs = minBackoffMs ?? defaultMinBackoffMs; + maxBackoffMs = maxBackoffMs ?? defaultMaxBackoffMs; + logWarning = logWarning ?? defaultLogWarning; + try { + return await callback(); + } catch (err: any) { + if (maxRetryCount <= 1 || minBackoffMs >= maxBackoffMs) { + throw err; + } + if (logWarning) { + logger.warn(err.message, `Retrying after (${minBackoffMs}ms). Remaining retries [${maxRetryCount}]`); + } + await sleep(minBackoffMs); + return await retrier(logger)(callback, maxRetryCount - 1, minBackoffMs * 2, maxBackoffMs, logWarning); + } + }; +}; diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index 73332b4..f969295 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -1,7 +1,9 @@ import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService, OnModuleInit, Optional } from '@nestjs/common'; -import ora from 'ora'; +import { promise as spinnerFor } from 'ora-classic'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; import { BeaconConfig, @@ -13,9 +15,10 @@ import { StateId, } from './response.interface'; import { ConfigService } from '../../config/config.service'; -import { PrometheusService } from '../../prometheus/prometheus.service'; +import { PrometheusService, TrackCLRequest } from '../../prometheus'; import { DownloadProgress } from '../../utils/download-progress/download-progress'; import { BaseRestProvider } from '../base/rest-provider'; +import { RequestOptions } from '../base/utils/func'; let ssz: typeof import('@lodestar/types').ssz; let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; @@ -47,6 +50,7 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { config.get('CL_API_URLS') as Array, config.get('CL_API_RESPONSE_TIMEOUT_MS'), config.get('CL_API_MAX_RETRIES'), + config.get('CL_API_RETRY_DELAY_MS'), logger, prometheus, ); @@ -74,25 +78,25 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { } public async getConfig(): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.config); + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.config)); const jsonBody = (await body.json()) as { data: BeaconConfig }; return jsonBody.data; } public async getGenesis(): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.genesis); + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.genesis)); const jsonBody = (await body.json()) as { data: GenesisResponse }; return jsonBody.data; } public async getBlockInfo(blockId: BlockId): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.blockInfo(blockId)); + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.blockInfo(blockId))); const jsonBody = (await body.json()) as { data: BlockInfoResponse }; return jsonBody.data; } public async getBeaconHeader(blockId: BlockId): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.beaconHeader(blockId)); + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.beaconHeader(blockId))); const jsonBody = (await body.json()) as { data: BlockHeaderResponse }; return jsonBody.data; } @@ -100,7 +104,9 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { public async getBeaconHeadersByParentRoot( parentRoot: RootHex, ): Promise<{ finalized: boolean; data: BlockHeaderResponse[] }> { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.beaconHeadersByParentRoot(parentRoot)); + const { body } = await this.retryRequest((baseUrl) => + this.baseGet(baseUrl, this.endpoints.beaconHeadersByParentRoot(parentRoot)), + ); return (await body.json()) as { finalized: boolean; data: BlockHeaderResponse[] }; } @@ -108,12 +114,14 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { stateId: StateId, signal?: AbortSignal, ): Promise<{ bodyBytes: Uint8Array; forkName: keyof typeof ForkName }> { - const requestPromise = this.baseGet(this.mainUrl, this.endpoints.state(stateId), { - signal, - headers: { accept: 'application/octet-stream' }, - }); + const requestPromise = this.retryRequest(async (baseUrl) => + this.baseGet(baseUrl, this.endpoints.state(stateId), { + signal, + headers: { accept: 'application/octet-stream' }, + }), + ); if (this.progress) { - ora.promise(requestPromise, { text: `Getting state response for state id [${stateId}]` }); + spinnerFor(requestPromise, { text: `Getting state response for state id [${stateId}]` }); } else { this.logger.log(`Getting state response for state id [${stateId}]`); } @@ -124,6 +132,15 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { return { bodyBytes, forkName }; } + @TrackCLRequest + protected baseGet( + baseUrl: string, + endpoint: string, + options?: RequestOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + return super.baseGet(baseUrl, endpoint, options); + } + public stateToView( bodyBytes: Uint8Array, forkName: keyof typeof ForkName, diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index be279d9..1ea50c1 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -6,8 +6,11 @@ import { connectTo } from 'stream-json/Assembler'; import { ELBlockSnapshot, ModuleKeys, ModuleKeysFind, Modules, Status } from './response.interface'; import { ConfigService } from '../../config/config.service'; -import { PrometheusService } from '../../prometheus/prometheus.service'; +import { PrometheusService, TrackKeysAPIRequest } from '../../prometheus'; import { BaseRestProvider } from '../base/rest-provider'; +import { RequestOptions } from '../base/utils/func'; +import BodyReadable from 'undici/types/readable'; +import { IncomingHttpHeaders } from 'undici/types/header'; @Injectable() export class Keysapi extends BaseRestProvider { @@ -27,6 +30,7 @@ export class Keysapi extends BaseRestProvider { config.get('KEYSAPI_API_URLS') as Array, config.get('KEYSAPI_API_RESPONSE_TIMEOUT_MS'), config.get('KEYSAPI_API_MAX_RETRIES'), + config.get('KEYSAPI_API_RETRY_DELAY_MS'), logger, prometheus, ); @@ -42,19 +46,23 @@ export class Keysapi extends BaseRestProvider { } public async getStatus(): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.status); + const { body } = await this.retryRequest( + (baseUrl) => this.baseGet(baseUrl, this.endpoints.status), + ) return (await body.json()) as Status; } public async getModules(): Promise { - const { body } = await this.baseGet(this.mainUrl, this.endpoints.modules); + const { body } = await this.retryRequest( + (baseUrl) => this.baseGet(baseUrl, this.endpoints.modules), + ); return (await body.json()) as Modules; } public async getModuleKeys(module_id: string | number, signal?: AbortSignal): Promise { - const resp = await this.baseGet(this.mainUrl, this.endpoints.moduleKeys(module_id), { - signal, - }); + const resp = await this.retryRequest( + (baseUrl) => this.baseGet(baseUrl, this.endpoints.moduleKeys(module_id), { signal }), + ); // TODO: ignore depositSignature ? const pipeline = chain([resp.body, parser()]); return await new Promise((resolve) => { @@ -67,10 +75,28 @@ export class Keysapi extends BaseRestProvider { keysToFind: string[], signal?: AbortSignal, ): Promise { - const { body } = await this.basePost(this.mainUrl, this.endpoints.findModuleKeys(module_id), { - pubkeys: keysToFind, - signal, - }); + const { body } = await this.retryRequest( + (baseUrl) => this.basePost(baseUrl, this.endpoints.findModuleKeys(module_id), { pubkeys: keysToFind, signal }), + ); return (await body.json()) as ModuleKeysFind; } + + @TrackKeysAPIRequest + protected baseGet( + baseUrl: string, + endpoint: string, + options?: RequestOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + return super.baseGet(baseUrl, endpoint, options); + } + + @TrackKeysAPIRequest + protected basePost( + baseUrl: string, + endpoint: string, + requestBody: any, + options?: RequestOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + return super.basePost(baseUrl, endpoint, requestBody, options); + } } From c8e5ca81ed33c47dfa363a3aee11ec02e09cda04 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:26:15 +0400 Subject: [PATCH 5/8] feat: prometheus metrics --- src/common/prometheus/index.ts | 4 + src/common/prometheus/interfaces/index.ts | 1 + .../interfaces/prometheus.interface.ts | 19 ++ src/common/prometheus/prometheus.constants.ts | 25 ++ .../prometheus/prometheus.controller.ts | 5 + src/common/prometheus/prometheus.module.ts | 13 + .../prometheus/prometheus.service.spec.ts | 19 -- src/common/prometheus/prometheus.service.ts | 230 +++++++++++++++++- src/common/providers/providers.module.ts | 34 ++- src/daemon/services/keys-indexer.ts | 50 +++- src/daemon/services/roots-processor.ts | 3 + src/daemon/services/roots-stack.ts | 65 ++++- 12 files changed, 434 insertions(+), 34 deletions(-) create mode 100644 src/common/prometheus/index.ts create mode 100644 src/common/prometheus/interfaces/index.ts create mode 100644 src/common/prometheus/interfaces/prometheus.interface.ts create mode 100644 src/common/prometheus/prometheus.constants.ts create mode 100644 src/common/prometheus/prometheus.controller.ts delete mode 100644 src/common/prometheus/prometheus.service.spec.ts diff --git a/src/common/prometheus/index.ts b/src/common/prometheus/index.ts new file mode 100644 index 0000000..2cf02ae --- /dev/null +++ b/src/common/prometheus/index.ts @@ -0,0 +1,4 @@ +export * from './prometheus.constants'; +export * from './prometheus.controller'; +export * from './prometheus.module'; +export * from './prometheus.service'; diff --git a/src/common/prometheus/interfaces/index.ts b/src/common/prometheus/interfaces/index.ts new file mode 100644 index 0000000..09b7631 --- /dev/null +++ b/src/common/prometheus/interfaces/index.ts @@ -0,0 +1 @@ +export * from './prometheus.interface'; diff --git a/src/common/prometheus/interfaces/prometheus.interface.ts b/src/common/prometheus/interfaces/prometheus.interface.ts new file mode 100644 index 0000000..36d8560 --- /dev/null +++ b/src/common/prometheus/interfaces/prometheus.interface.ts @@ -0,0 +1,19 @@ +import { Metrics } from '@willsoto/nestjs-prometheus'; +import * as client from 'prom-client'; +export { Metrics } from '@willsoto/nestjs-prometheus'; + +export type Options = + | client.GaugeConfiguration + | client.SummaryConfiguration + | client.CounterConfiguration + | client.HistogramConfiguration; + +export type Metric = T extends 'Gauge' + ? client.Gauge + : T extends 'Summary' + ? client.Summary + : T extends 'Counter' + ? client.Counter + : T extends 'Histogram' + ? client.Histogram + : never; diff --git a/src/common/prometheus/prometheus.constants.ts b/src/common/prometheus/prometheus.constants.ts new file mode 100644 index 0000000..bc68e91 --- /dev/null +++ b/src/common/prometheus/prometheus.constants.ts @@ -0,0 +1,25 @@ +export const APP_NAME = process.env.npm_package_name; +export const APP_DESCRIPTION = process.env.npm_package_description; + +export const METRICS_URL = '/metrics'; +export const METRICS_PREFIX = `${APP_NAME?.replace(/[- ]/g, '_')}_`; + +export const METRIC_BUILD_INFO = `build_info`; + +export const METRIC_OUTGOING_EL_REQUESTS_DURATION_SECONDS = `outgoing_el_requests_duration_seconds`; +export const METRIC_OUTGOING_EL_REQUESTS_COUNT = `outgoing_el_requests_count`; +export const METRIC_OUTGOING_CL_REQUESTS_DURATION_SECONDS = `outgoing_cl_requests_duration_seconds`; +export const METRIC_OUTGOING_CL_REQUESTS_COUNT = `outgoing_cl_requests_count`; +export const METRIC_OUTGOING_KEYSAPI_REQUESTS_DURATION_SECONDS = `outgoing_keysapi_requests_duration_seconds`; +export const METRIC_OUTGOING_KEYSAPI_REQUESTS_COUNT = `outgoing_keysapi_requests_count`; +export const METRIC_TASK_DURATION_SECONDS = `task_duration_seconds`; +export const METRIC_TASK_RESULT_COUNT = `task_result_count`; + +export const METRIC_DATA_ACTUALITY = `data_actuality`; +export const METRIC_LAST_PROCESSED_SLOT_NUMBER = `last_processed_slot_number`; +export const METRIC_ROOTS_STACK_SIZE = `roots_stack_size`; +export const METRIC_ROOTS_STACK_OLDEST_SLOT = `roots_stack_oldest_slot`; + +export const METRIC_KEYS_INDEXER_STORAGE_STATE_SLOT = `keys_indexer_storage_state_slot`; +export const METRIC_KEYS_INDEXER_ALL_VALIDATORS_COUNT = `keys_indexer_all_validators_count`; +export const METRIC_KEYS_CSM_VALIDATORS_COUNT = `keys_csm_validators_count`; diff --git a/src/common/prometheus/prometheus.controller.ts b/src/common/prometheus/prometheus.controller.ts new file mode 100644 index 0000000..9c01724 --- /dev/null +++ b/src/common/prometheus/prometheus.controller.ts @@ -0,0 +1,5 @@ +import { Controller } from '@nestjs/common'; +import { PrometheusController as PrometheusControllerSource } from '@willsoto/nestjs-prometheus'; + +@Controller() +export class PrometheusController extends PrometheusControllerSource {} diff --git a/src/common/prometheus/prometheus.module.ts b/src/common/prometheus/prometheus.module.ts index 29d2b37..a3a0805 100644 --- a/src/common/prometheus/prometheus.module.ts +++ b/src/common/prometheus/prometheus.module.ts @@ -1,9 +1,22 @@ import { Global, Module } from '@nestjs/common'; +import { PrometheusModule as PrometheusModuleSource } from '@willsoto/nestjs-prometheus'; +import { METRICS_PREFIX, METRICS_URL } from './prometheus.constants'; +import { PrometheusController } from './prometheus.controller'; import { PrometheusService } from './prometheus.service'; @Global() @Module({ + imports: [ + PrometheusModuleSource.register({ + controller: PrometheusController, + path: METRICS_URL, + defaultMetrics: { + enabled: true, + config: { prefix: METRICS_PREFIX }, + }, + }), + ], providers: [PrometheusService], exports: [PrometheusService], }) diff --git a/src/common/prometheus/prometheus.service.spec.ts b/src/common/prometheus/prometheus.service.spec.ts deleted file mode 100644 index a1e26ed..0000000 --- a/src/common/prometheus/prometheus.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { PrometheusService } from './prometheus.service'; - -describe('PrometheusService', () => { - let service: PrometheusService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [PrometheusService], - }).compile(); - - service = module.get(PrometheusService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/common/prometheus/prometheus.service.ts b/src/common/prometheus/prometheus.service.ts index 005e4d3..85b6dbe 100644 --- a/src/common/prometheus/prometheus.service.ts +++ b/src/common/prometheus/prometheus.service.ts @@ -1,4 +1,230 @@ -import { Injectable } from '@nestjs/common'; +import { LOGGER_PROVIDER, LoggerService } from '@lido-nestjs/logger'; +import { Inject, Injectable } from '@nestjs/common'; +import { Metrics, getOrCreateMetric } from '@willsoto/nestjs-prometheus'; +import { join } from 'lodash'; + +import { Metric, Options } from './interfaces'; +import { + METRICS_PREFIX, + METRIC_BUILD_INFO, + METRIC_OUTGOING_CL_REQUESTS_COUNT, + METRIC_OUTGOING_CL_REQUESTS_DURATION_SECONDS, + METRIC_OUTGOING_EL_REQUESTS_COUNT, + METRIC_OUTGOING_EL_REQUESTS_DURATION_SECONDS, + METRIC_OUTGOING_KEYSAPI_REQUESTS_COUNT, + METRIC_OUTGOING_KEYSAPI_REQUESTS_DURATION_SECONDS, + METRIC_TASK_DURATION_SECONDS, + METRIC_TASK_RESULT_COUNT, +} from './prometheus.constants'; +import { ConfigService } from '../config/config.service'; +import { WorkingMode } from '../config/env.validation'; + +export enum RequestStatus { + COMPLETE = 'complete', + ERROR = 'error', +} + +enum TaskStatus { + COMPLETE = 'complete', + ERROR = 'error', +} + +export function requestLabels(apiUrl: string, subUrl: string) { + const targetName = new URL(apiUrl).hostname; + const reqName = join( + subUrl + .split('?')[0] + .split('/') + .map((p) => { + if (p.includes('0x') || +p) return '{param}'; + return p; + }), + '/', + ); + return [targetName, reqName]; +} @Injectable() -export class PrometheusService {} +export class PrometheusService { + private prefix = METRICS_PREFIX; + + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + private config: ConfigService, + ) {} + + public getOrCreateMetric(type: T, options: Options): Metric { + const nameWithPrefix = this.prefix + options.name; + + return getOrCreateMetric(type, { + ...options, + name: nameWithPrefix, + }) as Metric; + } + + public buildInfo = this.getOrCreateMetric('Counter', { + name: METRIC_BUILD_INFO, + help: 'Build information', + labelNames: ['name', 'version', 'commit', 'branch', 'env', 'network'], + }); + + public outgoingELRequestsDuration = this.getOrCreateMetric('Histogram', { + name: METRIC_OUTGOING_EL_REQUESTS_DURATION_SECONDS, + help: 'Duration of outgoing execution layer requests', + buckets: [0.01, 0.1, 0.5, 1, 2, 5, 15, 30, 60], + labelNames: ['name', 'target'] as const, + }); + + public outgoingELRequestsCount = this.getOrCreateMetric('Gauge', { + name: METRIC_OUTGOING_EL_REQUESTS_COUNT, + help: 'Count of outgoing execution layer requests', + labelNames: ['name', 'target', 'status'] as const, + }); + + public outgoingCLRequestsDuration = this.getOrCreateMetric('Histogram', { + name: METRIC_OUTGOING_CL_REQUESTS_DURATION_SECONDS, + help: 'Duration of outgoing consensus layer requests', + buckets: [0.01, 0.1, 0.5, 1, 2, 5, 15, 30, 60], + labelNames: ['name', 'target'] as const, + }); + + public outgoingCLRequestsCount = this.getOrCreateMetric('Gauge', { + name: METRIC_OUTGOING_CL_REQUESTS_COUNT, + help: 'Count of outgoing consensus layer requests', + labelNames: ['name', 'target', 'status', 'code'] as const, + }); + + public outgoingKeysAPIRequestsDuration = this.getOrCreateMetric('Histogram', { + name: METRIC_OUTGOING_KEYSAPI_REQUESTS_DURATION_SECONDS, + help: 'Duration of outgoing KeysAPI requests', + buckets: [0.01, 0.1, 0.5, 1, 2, 5, 15, 30, 60], + labelNames: ['name', 'target'] as const, + }); + + public outgoingKeysAPIRequestsCount = this.getOrCreateMetric('Gauge', { + name: METRIC_OUTGOING_KEYSAPI_REQUESTS_COUNT, + help: 'Count of outgoing KeysAPI requests', + labelNames: ['name', 'target', 'status', 'code'] as const, + }); + + public taskDuration = this.getOrCreateMetric('Histogram', { + name: METRIC_TASK_DURATION_SECONDS, + help: 'Duration of task execution', + buckets: [5, 15, 30, 60, 120, 180, 240, 300, 400, 600], + labelNames: ['name'], + }); + + public taskCount = this.getOrCreateMetric('Gauge', { + name: METRIC_TASK_RESULT_COUNT, + help: 'Count of passed or failed tasks', + labelNames: ['name', 'status'], + }); +} + +export function TrackCLRequest(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const originalValue = descriptor.value; + descriptor.value = function (...args: any[]) { + if (this.config.get('WORKING_MODE') == WorkingMode.CLI) { + return originalValue.apply(this, args); + } + if (!this.prometheus) throw Error(`'${this.constructor.name}' class object must contain 'prometheus' property`); + const [apiUrl, subUrl] = args; + const [targetName, reqName] = requestLabels(apiUrl, subUrl); + const stop = this.prometheus.outgoingCLRequestsDuration.startTimer({ + name: reqName, + target: targetName, + }); + return originalValue + .apply(this, args) + .then((r: any) => { + this.prometheus.outgoingCLRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.COMPLETE, + code: 200, + }); + return r; + }) + .catch((e: any) => { + this.prometheus.outgoingCLRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.ERROR, + code: e.statusCode, + }); + throw e; + }) + .finally(() => stop()); + }; +} + +export function TrackKeysAPIRequest(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const originalValue = descriptor.value; + descriptor.value = function (...args: any[]) { + if (!this.prometheus) throw Error(`'${this.constructor.name}' class object must contain 'prometheus' property`); + const [apiUrl, subUrl] = args; + const [targetName, reqName] = requestLabels(apiUrl, subUrl); + const stop = this.prometheus.outgoingKeysAPIRequestsDuration.startTimer({ + name: reqName, + target: targetName, + }); + return originalValue + .apply(this, args) + .then((r: any) => { + this.prometheus.outgoingKeysAPIRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.COMPLETE, + code: 200, + }); + return r; + }) + .catch((e: any) => { + this.prometheus.outgoingKeysAPIRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.ERROR, + code: e.statusCode, + }); + throw e; + }) + .finally(() => stop()); + }; +} + +export function TrackTask(name: string) { + return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => { + const originalValue = descriptor.value; + + descriptor.value = function (...args: any[]) { + // "this" here will refer to the class instance + if (!this.prometheus) throw Error(`'${this.constructor.name}' class object must contain 'prometheus' property`); + const stop = this.prometheus.taskDuration.startTimer({ + name: name, + }); + this.logger.debug(`Task '${name}' in progress`); + return originalValue + .apply(this, args) + .then((r: any) => { + this.prometheus.taskCount.inc({ + name: name, + status: TaskStatus.COMPLETE, + }); + return r; + }) + .catch((e: Error) => { + this.logger.error(`Task '${name}' ended with an error`, e.stack); + this.prometheus.taskCount.inc({ + name: name, + status: TaskStatus.ERROR, + }); + throw e; + }) + .finally(() => { + const duration = stop(); + const used = process.memoryUsage().heapUsed / 1024 / 1024; + this.logger.debug(`Task '${name}' is complete. Used MB: ${used}. Duration: ${duration}`); + }); + }; + }; +} diff --git a/src/common/providers/providers.module.ts b/src/common/providers/providers.module.ts index 7aa6cde..aab80c4 100644 --- a/src/common/providers/providers.module.ts +++ b/src/common/providers/providers.module.ts @@ -7,17 +7,43 @@ import { Execution } from './execution/execution'; import { Keysapi } from './keysapi/keysapi'; import { ConfigService } from '../config/config.service'; import { WorkingMode } from '../config/env.validation'; -import { PrometheusService } from '../prometheus/prometheus.service'; +import { PrometheusService, RequestStatus } from '../prometheus'; import { UtilsModule } from '../utils/utils.module'; const ExecutionDaemon = () => FallbackProviderModule.forRootAsync({ - async useFactory(configService: ConfigService) { + async useFactory(configService: ConfigService, prometheusService: PrometheusService) { return { urls: configService.get('EL_RPC_URLS') as NonEmptyArray, network: configService.get('ETH_NETWORK'), - // TODO: add prometheus metrics - // fetchMiddlewares: [ ... ], + fetchMiddlewares: [ + async (next, ctx) => { + const targetName = new URL(ctx.provider.connection.url).hostname; + const reqName = 'batch'; + const stop = prometheusService.outgoingELRequestsDuration.startTimer({ + name: reqName, + target: targetName, + }); + return await next() + .then((r: any) => { + prometheusService.outgoingELRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.COMPLETE, + }); + return r; + }) + .catch((e: any) => { + prometheusService.outgoingELRequestsCount.inc({ + name: reqName, + target: targetName, + status: RequestStatus.ERROR, + }); + throw e; + }) + .finally(() => stop()); + }, + ], }; }, inject: [ConfigService, PrometheusService], diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index e3a91e7..7f76b96 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -4,9 +4,16 @@ import { ListCompositeTreeView } from '@chainsafe/ssz/lib/view/listComposite'; import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { Inject, Injectable, LoggerService, OnModuleInit } from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnApplicationBootstrap, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '../../common/config/config.service'; +import { + METRIC_KEYS_CSM_VALIDATORS_COUNT, + METRIC_KEYS_INDEXER_ALL_VALIDATORS_COUNT, + METRIC_KEYS_INDEXER_STORAGE_STATE_SLOT, + PrometheusService, + TrackTask, +} from '../../common/prometheus'; import { toHex } from '../../common/prover/helpers/proofs'; import { KeyInfo } from '../../common/prover/types'; import { Consensus } from '../../common/providers/consensus/consensus'; @@ -52,7 +59,7 @@ function Single(target: any, propertyKey: string, descriptor: PropertyDescriptor } @Injectable() -export class KeysIndexer implements OnModuleInit { +export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap { private startedAt: number = 0; private info: Low; @@ -61,6 +68,7 @@ export class KeysIndexer implements OnModuleInit { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly config: ConfigService, + protected readonly prometheus: PrometheusService, protected readonly consensus: Consensus, protected readonly keysapi: Keysapi, ) {} @@ -69,6 +77,10 @@ export class KeysIndexer implements OnModuleInit { await this.initOrReadServiceData(); } + public async onApplicationBootstrap(): Promise { + this.setMetrics(); + } + public getKey = (valIndex: number): KeyInfo | undefined => { return this.storage.data[valIndex]; }; @@ -90,6 +102,7 @@ export class KeysIndexer implements OnModuleInit { .finally(() => (this.startedAt = 0)); } + @TrackTask('update-keys-indexer') private async baseRun( stateRoot: RootHex, finalizedSlot: Slot, @@ -164,11 +177,11 @@ export class KeysIndexer implements OnModuleInit { lastValidatorsCount: 0, }; this.info = new Low( - new JSONFile('.keys-indexer-info.json'), + new JSONFile('storage/.keys-indexer-info.json'), defaultInfo, ); this.storage = new Low( - new JSONFile('.keys-indexer-storage.json'), + new JSONFile('storage/.keys-indexer-storage.json'), {}, ); await this.info.read(); @@ -257,4 +270,33 @@ export class KeysIndexer implements OnModuleInit { } } }; + + private setMetrics() { + const info = () => this.info.data; + const keysCount = () => Object.keys(this.storage.data).length; + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_KEYS_INDEXER_STORAGE_STATE_SLOT, + help: 'Keys indexer storage state slot', + labelNames: [], + collect() { + this.set(info().storageStateSlot); + }, + }); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_KEYS_INDEXER_ALL_VALIDATORS_COUNT, + help: 'Keys indexer all validators count', + labelNames: [], + collect() { + this.set(info().lastValidatorsCount); + }, + }); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_KEYS_CSM_VALIDATORS_COUNT, + help: 'Keys indexer CSM validators count', + labelNames: [], + collect() { + this.set(keysCount()); + }, + }); + } } diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index 3048e0f..a0b2f6d 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -3,6 +3,7 @@ import { Inject, Injectable, LoggerService } from '@nestjs/common'; import { KeysIndexer } from './keys-indexer'; import { RootSlot, RootsStack } from './roots-stack'; +import { PrometheusService, TrackTask } from '../../common/prometheus'; import { ProverService } from '../../common/prover/prover.service'; import { Consensus } from '../../common/providers/consensus/consensus'; import { BlockHeaderResponse, RootHex } from '../../common/providers/consensus/response.interface'; @@ -11,12 +12,14 @@ import { BlockHeaderResponse, RootHex } from '../../common/providers/consensus/r export class RootsProcessor { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly prometheus: PrometheusService, protected readonly consensus: Consensus, protected readonly keysIndexer: KeysIndexer, protected readonly rootsStack: RootsStack, protected readonly prover: ProverService, ) {} + @TrackTask('process-root') public async process(blockRootToProcess: RootHex, finalizedHeader: BlockHeaderResponse): Promise { this.logger.log(`🛃 Root in processing [${blockRootToProcess}]`); const blockInfoToProcess = await this.consensus.getBlockInfo(blockRootToProcess); diff --git a/src/daemon/services/roots-stack.ts b/src/daemon/services/roots-stack.ts index cc24611..4986450 100644 --- a/src/daemon/services/roots-stack.ts +++ b/src/daemon/services/roots-stack.ts @@ -1,8 +1,16 @@ import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, OnApplicationBootstrap, OnModuleInit } from '@nestjs/common'; import { KeysIndexer } from './keys-indexer'; +import { + METRIC_DATA_ACTUALITY, + METRIC_LAST_PROCESSED_SLOT_NUMBER, + METRIC_ROOTS_STACK_OLDEST_SLOT, + METRIC_ROOTS_STACK_SIZE, + PrometheusService, +} from '../../common/prometheus'; +import { Consensus } from '../../common/providers/consensus/consensus'; import { RootHex } from '../../common/providers/consensus/response.interface'; export type RootSlot = { blockRoot: RootHex; slotNumber: number }; @@ -14,16 +22,24 @@ type RootsStackServiceInfo = { type RootsStackServiceStorage = { [slot: number]: RootHex }; @Injectable() -export class RootsStack implements OnModuleInit { +export class RootsStack implements OnModuleInit, OnApplicationBootstrap { private info: Low; private storage: Low; - constructor(protected readonly keysIndexer: KeysIndexer) {} + constructor( + protected readonly prometheus: PrometheusService, + protected readonly keysIndexer: KeysIndexer, + protected readonly consensus: Consensus, + ) {} async onModuleInit(): Promise { await this.initOrReadServiceData(); } + async onApplicationBootstrap(): Promise { + this.setMetrics(); + } + public getNextEligible(): RootSlot | undefined { for (const slot in this.storage.data) { if (this.keysIndexer.isTrustedForAnyDuty(Number(slot))) { @@ -54,11 +70,50 @@ export class RootsStack implements OnModuleInit { } private async initOrReadServiceData() { - this.info = new Low(new JSONFile('.roots-stack-info.json'), { + this.info = new Low(new JSONFile('storage/.roots-stack-info.json'), { lastProcessedRootSlot: undefined, }); - this.storage = new Low(new JSONFile('.roots-stack-storage.json'), {}); + this.storage = new Low(new JSONFile('storage/.roots-stack-storage.json'), {}); await this.info.read(); await this.storage.read(); } + + private setMetrics() { + const lastProcessed = () => Number(this.info.data.lastProcessedRootSlot?.slotNumber); + const getSlotTimeDiffWithNow = () => Date.now() - this.consensus.slotToTimestamp(lastProcessed()) * 1000; + const rootsStackSize = () => Object.keys(this.storage.data).length; + const rootsStackOldestSlot = () => Math.min(...Object.keys(this.storage.data).map(Number)); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_DATA_ACTUALITY, + help: 'Data actuality', + labelNames: [], + collect() { + this.set(getSlotTimeDiffWithNow()); + }, + }); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_LAST_PROCESSED_SLOT_NUMBER, + help: 'Last processed slot', + labelNames: [], + collect() { + this.set(lastProcessed()); + }, + }); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_ROOTS_STACK_SIZE, + help: 'Roots stack size', + labelNames: [], + collect() { + this.set(rootsStackSize()); + }, + }); + this.prometheus.getOrCreateMetric('Gauge', { + name: METRIC_ROOTS_STACK_OLDEST_SLOT, + help: 'Roots stack oldest slot', + labelNames: [], + collect() { + this.set(rootsStackOldestSlot()); + }, + }); + } } From b2945ec5558663968696aed4465966b07236d727 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:28:57 +0400 Subject: [PATCH 6/8] chore: empty tests --- src/cli/cli.service.spec.ts | 21 ------------------ src/common/config/config.service.spec.ts | 19 ---------------- src/common/health/health.service.spec.ts | 19 ---------------- src/common/prover/prover.service.spec.ts | 19 ---------------- .../providers/consensus/consensus.spec.ts | 19 ---------------- .../providers/execution/execution.spec.ts | 19 ---------------- src/common/providers/keysapi/keysapi.spec.ts | 19 ---------------- .../download-progress.spec.ts | 19 ---------------- src/daemon/daemon.service.spec.ts | 21 ------------------ src/daemon/daemon.service.ts | 8 ++----- src/main.ts | 2 ++ test/cli.e2e-spec.ts | 22 ++++++++++++++----- test/daemon.e2e-spec.ts | 4 ++++ test/{jest-e2e.json => jest-cli-e2e.json} | 2 +- test/jest-daemon-e2e.json | 9 ++++++++ 15 files changed, 35 insertions(+), 187 deletions(-) delete mode 100644 src/cli/cli.service.spec.ts delete mode 100644 src/common/config/config.service.spec.ts delete mode 100644 src/common/health/health.service.spec.ts delete mode 100644 src/common/prover/prover.service.spec.ts delete mode 100644 src/common/providers/consensus/consensus.spec.ts delete mode 100644 src/common/providers/execution/execution.spec.ts delete mode 100644 src/common/providers/keysapi/keysapi.spec.ts delete mode 100644 src/common/utils/download-progress/download-progress.spec.ts delete mode 100644 src/daemon/daemon.service.spec.ts rename test/{jest-e2e.json => jest-cli-e2e.json} (81%) create mode 100644 test/jest-daemon-e2e.json diff --git a/src/cli/cli.service.spec.ts b/src/cli/cli.service.spec.ts deleted file mode 100644 index 1186d7c..0000000 --- a/src/cli/cli.service.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { CliService } from './cli.service'; -import { LoggerModule } from '../common/logger/logger.module'; - -describe('CliService', () => { - let service: CliService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [LoggerModule], - providers: [CliService], - }).compile(); - - service = module.get(CliService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/common/config/config.service.spec.ts b/src/common/config/config.service.spec.ts deleted file mode 100644 index 26db728..0000000 --- a/src/common/config/config.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { ConfigService } from './config.service'; - -describe('ConfigService', () => { - let service: ConfigService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ConfigService], - }).compile(); - - service = module.get(ConfigService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/common/health/health.service.spec.ts b/src/common/health/health.service.spec.ts deleted file mode 100644 index 9631fa8..0000000 --- a/src/common/health/health.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { HealthService } from './health.service'; - -describe('HealthService', () => { - let service: HealthService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [HealthService], - }).compile(); - - service = module.get(HealthService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/common/prover/prover.service.spec.ts b/src/common/prover/prover.service.spec.ts deleted file mode 100644 index ef6a756..0000000 --- a/src/common/prover/prover.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { ProverService } from './prover.service'; - -describe('HandlersService', () => { - let service: ProverService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ProverService], - }).compile(); - - service = module.get(ProverService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/common/providers/consensus/consensus.spec.ts b/src/common/providers/consensus/consensus.spec.ts deleted file mode 100644 index 11c3261..0000000 --- a/src/common/providers/consensus/consensus.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { Consensus } from './consensus'; - -describe('Consensus', () => { - let provider: Consensus; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [Consensus], - }).compile(); - - provider = module.get(Consensus); - }); - - it('should be defined', () => { - expect(provider).toBeDefined(); - }); -}); diff --git a/src/common/providers/execution/execution.spec.ts b/src/common/providers/execution/execution.spec.ts deleted file mode 100644 index 1c873be..0000000 --- a/src/common/providers/execution/execution.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { Execution } from './execution'; - -describe('Execution', () => { - let provider: Execution; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [Execution], - }).compile(); - - provider = module.get(Execution); - }); - - it('should be defined', () => { - expect(provider).toBeDefined(); - }); -}); diff --git a/src/common/providers/keysapi/keysapi.spec.ts b/src/common/providers/keysapi/keysapi.spec.ts deleted file mode 100644 index 3f745b3..0000000 --- a/src/common/providers/keysapi/keysapi.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { Keysapi } from './keysapi'; - -describe('Keysapi', () => { - let provider: Keysapi; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [Keysapi], - }).compile(); - - provider = module.get(Keysapi); - }); - - it('should be defined', () => { - expect(provider).toBeDefined(); - }); -}); diff --git a/src/common/utils/download-progress/download-progress.spec.ts b/src/common/utils/download-progress/download-progress.spec.ts deleted file mode 100644 index b30bfa5..0000000 --- a/src/common/utils/download-progress/download-progress.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { DownloadProgress } from './download-progress'; - -describe('DownloadProgress', () => { - let provider: DownloadProgress; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [DownloadProgress], - }).compile(); - - provider = module.get(DownloadProgress); - }); - - it('should be defined', () => { - expect(provider).toBeDefined(); - }); -}); diff --git a/src/daemon/daemon.service.spec.ts b/src/daemon/daemon.service.spec.ts deleted file mode 100644 index ba33c5b..0000000 --- a/src/daemon/daemon.service.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { DaemonService } from './daemon.service'; -import { LoggerModule } from '../common/logger/logger.module'; - -describe('DaemonService', () => { - let service: DaemonService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [LoggerModule], - providers: [DaemonService], - }).compile(); - - service = module.get(DaemonService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/daemon/daemon.service.ts b/src/daemon/daemon.service.ts index 3193164..b46b5b0 100644 --- a/src/daemon/daemon.service.ts +++ b/src/daemon/daemon.service.ts @@ -9,7 +9,7 @@ import { ConfigService } from '../common/config/config.service'; import { Consensus } from '../common/providers/consensus/consensus'; @Injectable() -export class DaemonService implements OnModuleInit, OnApplicationBootstrap { +export class DaemonService implements OnModuleInit { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly config: ConfigService, @@ -23,11 +23,7 @@ export class DaemonService implements OnModuleInit, OnApplicationBootstrap { this.logger.log('Working mode: DAEMON'); } - async onApplicationBootstrap() { - this.loop().then(); - } - - private async loop() { + public async loop() { while (true) { try { await this.baseRun(); diff --git a/src/main.ts b/src/main.ts index 0e9fa1d..b4e8490 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,6 +5,7 @@ import { CliModule } from './cli/cli.module'; import { ConfigService } from './common/config/config.service'; import { WorkingMode } from './common/config/env.validation'; import { DaemonModule } from './daemon/daemon.module'; +import { DaemonService } from './daemon/daemon.service'; async function bootstrapCLI() { process @@ -21,6 +22,7 @@ async function bootstrapDaemon() { const daemonApp = await NestFactory.create(DaemonModule, { logger: false }); // disable initialising logs from NestJS const configService: ConfigService = daemonApp.get(ConfigService); await daemonApp.listen(configService.get('HTTP_PORT'), '0.0.0.0'); + daemonApp.get(DaemonService).loop().then(); } async function bootstrap() { diff --git a/test/cli.e2e-spec.ts b/test/cli.e2e-spec.ts index e5840e3..ae22f01 100644 --- a/test/cli.e2e-spec.ts +++ b/test/cli.e2e-spec.ts @@ -1,18 +1,30 @@ import { INestApplication } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { CommandTestFactory } from 'nest-commander-testing'; import { CliModule } from '../src/cli/cli.module'; +import { ConfigService } from '../src/common/config/config.service'; +import { EnvironmentVariables } from '../src/common/config/env.validation'; + +class CustomConfigService extends ConfigService { + public get(key: T): EnvironmentVariables[T] { + if (key == 'WORKING_MODE') { + return 'cli' as EnvironmentVariables[T]; + } + return super.get(key) as EnvironmentVariables[T]; + } +} describe('Cli (e2e)', () => { let app: INestApplication; beforeEach(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ + const moduleFixture: TestingModule = await CommandTestFactory.createTestingCommand({ imports: [CliModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - await app.init(); + }) + .overrideProvider(ConfigService) + .useClass(CustomConfigService) + .compile(); }); it('does nothing', () => { diff --git a/test/daemon.e2e-spec.ts b/test/daemon.e2e-spec.ts index d480641..8ff53d3 100644 --- a/test/daemon.e2e-spec.ts +++ b/test/daemon.e2e-spec.ts @@ -15,6 +15,10 @@ describe('Daemon (e2e)', () => { await app.init(); }); + afterEach(async () => { + await app.close(); + }); + it('does nothing', () => { return; }); diff --git a/test/jest-e2e.json b/test/jest-cli-e2e.json similarity index 81% rename from test/jest-e2e.json rename to test/jest-cli-e2e.json index ba626ac..b671bd6 100644 --- a/test/jest-e2e.json +++ b/test/jest-cli-e2e.json @@ -2,7 +2,7 @@ "moduleFileExtensions": ["js", "json", "ts"], "rootDir": ".", "testEnvironment": "node", - "testRegex": ".e2e-spec.ts$", + "testRegex": "cli.e2e-spec.ts$", "transform": { "^.+\\.(t|j)s?$": ["@swc/jest"] } diff --git a/test/jest-daemon-e2e.json b/test/jest-daemon-e2e.json new file mode 100644 index 0000000..e82dbb3 --- /dev/null +++ b/test/jest-daemon-e2e.json @@ -0,0 +1,9 @@ +{ + "moduleFileExtensions": ["js", "json", "ts"], + "rootDir": ".", + "testEnvironment": "node", + "testRegex": "daemon.e2e-spec.ts$", + "transform": { + "^.+\\.(t|j)s?$": ["@swc/jest"] + } +} From 4c082da12767e3f9b5635d350e5d14dd1bf8f1f8 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:30:06 +0400 Subject: [PATCH 7/8] feat: docker and etc. --- .env.example | 4 ++ .gitignore | 5 +- .nvmrc | 2 +- Dockerfile.cli | 22 +++++++ Dockerfile.daemon | 26 ++++++++ docker-compose.yml | 74 +++++++++++++++++++++++ package.json | 16 ++--- prometheus.yml | 13 ++++ src/common/health/health.constants.ts | 1 + src/common/health/health.controller.ts | 23 +++++++ src/common/health/health.module.ts | 7 ++- src/common/health/health.service.ts | 4 -- src/common/health/index.ts | 3 + yarn.lock | 83 +++++++++++++++++++++++++- 14 files changed, 265 insertions(+), 18 deletions(-) create mode 100644 Dockerfile.cli create mode 100644 Dockerfile.daemon create mode 100644 docker-compose.yml create mode 100644 prometheus.yml create mode 100644 src/common/health/health.constants.ts create mode 100644 src/common/health/health.controller.ts delete mode 100644 src/common/health/health.service.ts create mode 100644 src/common/health/index.ts diff --git a/.env.example b/.env.example index 557c1b9..13ea7a7 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,10 @@ +# both daemon and cli work mode ETH_NETWORK=1 EL_RPC_URLS=https://mainnet.infura.io/v3/... CL_API_URLS=https://quiknode.pro/... CSM_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320 VERIFIER_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6321 TX_SIGNER_PRIVATE_KEY=0x... + +# only for daemon mode +KEYSAPI_API_URLS=https://keys-api.lido.fi/ diff --git a/.gitignore b/.gitignore index 693c23e..efff06e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,8 @@ # ENV /.env -# Storage -/.keys-indexer-* -/.roots-stack-* +# Application storage +/storage # Logs logs diff --git a/.nvmrc b/.nvmrc index 7ea6a59..020fc41 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.11.0 +v20.12.1 diff --git a/Dockerfile.cli b/Dockerfile.cli new file mode 100644 index 0000000..89a9ef3 --- /dev/null +++ b/Dockerfile.cli @@ -0,0 +1,22 @@ +FROM node:20.12.1-alpine as building + +WORKDIR /app + +COPY package.json yarn.lock ./ +COPY ./tsconfig*.json ./ +COPY ./src ./src + +RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean && yarn typechain +RUN yarn build + +FROM node:20.12.1-alpine + +WORKDIR /app + +COPY --from=building /app/dist ./dist +COPY --from=building /app/node_modules ./node_modules +COPY ./package.json ./ + +USER node + +ENTRYPOINT ["yarn"] diff --git a/Dockerfile.daemon b/Dockerfile.daemon new file mode 100644 index 0000000..95ac397 --- /dev/null +++ b/Dockerfile.daemon @@ -0,0 +1,26 @@ +FROM node:20.12.1-alpine as building + +WORKDIR /app + +COPY package.json yarn.lock ./ +COPY ./tsconfig*.json ./ +COPY ./src ./src + +RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean && yarn typechain +RUN yarn build + +FROM node:20.12.1-alpine + +WORKDIR /app + +COPY --from=building /app/dist ./dist +COPY --from=building /app/node_modules ./node_modules +COPY ./package.json ./ +RUN mkdir -p ./storage/ && chown -R node:node ./storage/ + +USER node + +HEALTHCHECK --interval=60s --timeout=10s --retries=3 \ + CMD sh -c "wget -nv -t1 --spider http://localhost:$HTTP_PORT/health" || exit 1 + +CMD ["yarn", "start:prod"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..df19b4f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,74 @@ +services: + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - --config.file=/etc/prometheus/prometheus.yml + + daemon: + build: + context: . + dockerfile: ./Dockerfile.daemon + container_name: prover-daemon + restart: unless-stopped + environment: + - DRY_RUN=${DRY_RUN:-false} + - ETH_NETWORK=${ETH_NETWORK} + - KEYSAPI_API_URLS=${KEYSAPI_API_URLS} + - EL_RPC_URLS=${EL_RPC_URLS} + - CL_API_URLS=${CL_API_URLS} + - CSM_ADDRESS=${CSM_ADDRESS} + - VERIFIER_ADDRESS=${VERIFIER_ADDRESS} + - TX_SIGNER_PRIVATE_KEY=${TX_SIGNER_PRIVATE_KEY:-} + expose: + - "${HTTP_PORT:-8080}" + ports: + - "${EXTERNAL_HTTP_PORT:-${HTTP_PORT:-8080}}:${HTTP_PORT:-8080}" + volumes: + - ./storage/:/app/storage/ + depends_on: + - prometheus + + # + # CLI tools + # + withdrawal: + build: + context: . + dockerfile: ./Dockerfile.cli + entrypoint: + - yarn + - withdrawal + container_name: prover-cli-withdrawal + restart: no + environment: + - DRY_RUN=${DRY_RUN:-false} + - ETH_NETWORK=${ETH_NETWORK} + - EL_RPC_URLS=${EL_RPC_URLS} + - CL_API_URLS=${CL_API_URLS} + - CSM_ADDRESS=${CSM_ADDRESS} + - VERIFIER_ADDRESS=${VERIFIER_ADDRESS} + - TX_SIGNER_PRIVATE_KEY=${TX_SIGNER_PRIVATE_KEY:-} + + slashing: + build: + context: . + dockerfile: ./Dockerfile.cli + entrypoint: + - yarn + - slashing + container_name: prover-cli-slashing + restart: no + environment: + - DRY_RUN=${DRY_RUN:-false} + - ETH_NETWORK=${ETH_NETWORK} + - EL_RPC_URLS=${EL_RPC_URLS} + - CL_API_URLS=${CL_API_URLS} + - CSM_ADDRESS=${CSM_ADDRESS} + - VERIFIER_ADDRESS=${VERIFIER_ADDRESS} + - TX_SIGNER_PRIVATE_KEY=${TX_SIGNER_PRIVATE_KEY:-} diff --git a/package.json b/package.json index f1c3da4..0d33aa7 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "private": true, "license": "GPL-3.0", "scripts": { - "prove": "nest build && NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node dist/main prove", - "prove:debug": "nest build && NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node --inspect dist/main prove", + "prove": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node dist/main prove", + "prove:debug": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node --inspect dist/main prove", "slashing": "yarn prove slashing", "slashing:debug": "yarn prove:debug slashing", "withdrawal": "yarn prove withdrawal", @@ -19,14 +19,12 @@ "start": "nest start", "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", + "start:prod": "NODE_OPTIONS=--max_old_space_size=8192 node dist/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"", "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config ./test/jest-e2e.json" + "test-daemon": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest --config test/jest-daemon-e2e.json", + "test-cli": "WORKING_MODE=cli NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest --config test/jest-cli-e2e.json" }, "dependencies": { "@huanshiwushuang/lowdb": "^6.0.2", @@ -39,14 +37,18 @@ "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "@nestjs/terminus": "^10.2.3", "@typechain/ethers-v5": "^11.1.2", "@types/cli-progress": "^3.11.5", + "@willsoto/nestjs-prometheus": "^6.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "ethers": "^5.7.2", "nest-commander": "^3.12.5", + "nest-commander-testing": "^3.3.0", "nest-winston": "^1.9.4", "ora-classic": "^5.4.2", + "prom-client": "^15.1.1", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1", "stream-chain": "^2.2.5", diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..316c0fb --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 10s + evaluation_interval: 15s + +scrape_configs: + + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'daemon' + static_configs: + - targets: ['daemon:8080'] diff --git a/src/common/health/health.constants.ts b/src/common/health/health.constants.ts new file mode 100644 index 0000000..ac47243 --- /dev/null +++ b/src/common/health/health.constants.ts @@ -0,0 +1 @@ +export const HEALTH_URL = 'health'; diff --git a/src/common/health/health.controller.ts b/src/common/health/health.controller.ts new file mode 100644 index 0000000..129026f --- /dev/null +++ b/src/common/health/health.controller.ts @@ -0,0 +1,23 @@ +import * as v8 from 'v8'; + +import { Controller, Get } from '@nestjs/common'; +import { HealthCheck, HealthCheckService, MemoryHealthIndicator } from '@nestjs/terminus'; + +import { HEALTH_URL } from './health.constants'; + +@Controller(HEALTH_URL) +export class HealthController { + private readonly maxHeapSize: number; + constructor( + private health: HealthCheckService, + private memory: MemoryHealthIndicator, + ) { + this.maxHeapSize = v8.getHeapStatistics().heap_size_limit; + } + + @Get() + @HealthCheck() + check() { + return this.health.check([async () => this.memory.checkHeap('memoryHeap', this.maxHeapSize)]); + } +} diff --git a/src/common/health/health.module.ts b/src/common/health/health.module.ts index 6b4fcf9..ca8bca8 100644 --- a/src/common/health/health.module.ts +++ b/src/common/health/health.module.ts @@ -1,8 +1,11 @@ import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; -import { HealthService } from './health.service'; +import { HealthController } from './health.controller'; @Module({ - providers: [HealthService], + providers: [], + controllers: [HealthController], + imports: [TerminusModule], }) export class HealthModule {} diff --git a/src/common/health/health.service.ts b/src/common/health/health.service.ts deleted file mode 100644 index a329ae7..0000000 --- a/src/common/health/health.service.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -@Injectable() -export class HealthService {} diff --git a/src/common/health/index.ts b/src/common/health/index.ts new file mode 100644 index 0000000..4d9fb77 --- /dev/null +++ b/src/common/health/index.ts @@ -0,0 +1,3 @@ +export * from './health.constants'; +export * from './health.controller'; +export * from './health.module'; diff --git a/yarn.lock b/yarn.lock index 199f03b..be72570 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1258,6 +1258,14 @@ jsonc-parser "3.2.0" pluralize "8.0.0" +"@nestjs/terminus@^10.2.3": + version "10.2.3" + resolved "https://registry.yarnpkg.com/@nestjs/terminus/-/terminus-10.2.3.tgz#72c8a66d04df52aeaae807551245480fd7239a75" + integrity sha512-iX7gXtAooePcyQqFt57aDke5MzgdkBeYgF5YsFNNFwOiAFdIQEhfv3PR0G+HlH9F6D7nBCDZt9U87Pks/qHijg== + dependencies: + boxen "5.1.2" + check-disk-space "3.4.0" + "@nestjs/testing@^10.0.0": version "10.3.1" resolved "https://registry.yarnpkg.com/@nestjs/testing/-/testing-10.3.1.tgz#ea28a7d29122dd3a2df1542842e741a52dd7c474" @@ -1300,6 +1308,11 @@ consola "^2.15.0" node-fetch "^2.6.1" +"@opentelemetry/api@^1.4.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" + integrity sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w== + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" @@ -1998,6 +2011,11 @@ "@webassemblyjs/ast" "1.11.6" "@xtuc/long" "4.2.2" +"@willsoto/nestjs-prometheus@^6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@willsoto/nestjs-prometheus/-/nestjs-prometheus-6.0.0.tgz#6ef4d5d5dfb04ebe982aab6f3a7893974e89a669" + integrity sha512-Krmda5CT9xDPjab8Eqdqiwi7xkZSX60A5rEGVLEDjUG6J6Rw5SCZ/BPaRk+MxNGWzUrRkM7K5FtTg38vWIOt1Q== + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -2073,6 +2091,13 @@ ajv@^6.12.4, ajv@^6.12.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ansi-align@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59" + integrity sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w== + dependencies: + string-width "^4.1.0" + ansi-colors@4.1.3: version "4.1.3" resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" @@ -2379,6 +2404,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -2434,6 +2464,20 @@ body-parser@1.20.2: type-is "~1.6.18" unpipe "1.0.0" +boxen@5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" + integrity sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ== + dependencies: + ansi-align "^3.0.0" + camelcase "^6.2.0" + chalk "^4.1.0" + cli-boxes "^2.2.1" + string-width "^4.2.2" + type-fest "^0.20.2" + widest-line "^3.1.0" + wrap-ansi "^7.0.0" + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -2589,6 +2633,11 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== +check-disk-space@3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/check-disk-space/-/check-disk-space-3.4.0.tgz#eb8e69eee7a378fd12e35281b8123a8b4c4a8ff7" + integrity sha512-drVkSqfwA+TvuEhFipiR1OC9boEGZL5RrWvVsOthdcvQNXyCCuKkEiTOTXZ7qxSf/GLwq4GvzfrQD/Wz325hgw== + chokidar@3.5.3, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" @@ -2633,6 +2682,11 @@ class-validator@^0.14.1: libphonenumber-js "^1.10.53" validator "^13.9.0" +cli-boxes@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" + integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== + cli-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" @@ -5338,6 +5392,11 @@ neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== +nest-commander-testing@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/nest-commander-testing/-/nest-commander-testing-3.3.0.tgz#3180d3e51555cf00ea41256fa67a37a12202201e" + integrity sha512-XZ+5xMo4gnZc2xdkug7hTe7CTS0FhggzQrbeDo783CLfpS185iiLdTaENcbHD8xuyysgwXiSok3vQm5Cht8Mww== + nest-commander@^3.12.5: version "3.12.5" resolved "https://registry.yarnpkg.com/nest-commander/-/nest-commander-3.12.5.tgz#d9a98b101dd21aad0d30df009d3a2868db27ca57" @@ -5761,6 +5820,14 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +prom-client@^15.1.1: + version "15.1.1" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.1.tgz#71ba84371241acd173181b04a436782c246f3652" + integrity sha512-GVA2H96QCg2q71rjc3VYvSrVG7OpnJxyryC7dMzvfJfpJJHzQVwF3TJLfHzKORcwJpElWs1TwXLthlJAFJxq2A== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -6345,7 +6412,7 @@ string-length@^4.0.1: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -6541,6 +6608,13 @@ tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + terser-webpack-plugin@^5.3.10: version "5.3.10" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199" @@ -7061,6 +7135,13 @@ which@^2.0.1: dependencies: isexe "^2.0.0" +widest-line@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca" + integrity sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg== + dependencies: + string-width "^4.0.0" + winston-transport@^4.5.0: version "4.7.0" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.7.0.tgz#e302e6889e6ccb7f383b926df6936a5b781bd1f0" From e7552d4a07537ee6babd73c7f7eedc14541f3837 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 10 Apr 2024 10:37:21 +0400 Subject: [PATCH 8/8] fix: linter --- src/common/providers/base/rest-provider.ts | 34 ++++++++++++++++------ src/common/providers/keysapi/keysapi.ts | 20 +++++-------- src/daemon/daemon.service.ts | 2 +- test/cli.e2e-spec.ts | 6 +--- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/common/providers/base/rest-provider.ts b/src/common/providers/base/rest-provider.ts index 700ee6f..41c4d99 100644 --- a/src/common/providers/base/rest-provider.ts +++ b/src/common/providers/base/rest-provider.ts @@ -6,10 +6,17 @@ import BodyReadable from 'undici/types/readable'; import { RequestOptions, RequestPolicy, rejectDelay, retrier } from './utils/func'; import { PrometheusService } from '../../prometheus'; -export type RetryOptions = RequestOptions & RequestPolicy & { useFallbackOnRejected?: (err: Error, current_error: Error) => boolean; useFallbackOnResolved?: (data: any) => boolean} +export type RetryOptions = RequestOptions & + RequestPolicy & { + useFallbackOnRejected?: (err: Error, current_error: Error) => boolean; + useFallbackOnResolved?: (data: any) => boolean; + }; class RequestError extends Error { - constructor(message: string, public readonly statusCode?: number) { + constructor( + message: string, + public readonly statusCode?: number, + ) { super(message); } } @@ -35,13 +42,16 @@ export abstract class BaseRestProvider { } protected async retryRequest( - callback: (apiURL: string, options?: RequestOptions) => Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }>, + callback: ( + apiURL: string, + options?: RequestOptions, + ) => Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }>, options?: RetryOptions, ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { options = { ...this.requestPolicy, - useFallbackOnRejected: (() => true), // use fallback on error as default - useFallbackOnResolved: (() => false), // do NOT use fallback on success as default + useFallbackOnRejected: () => true, // use fallback on error as default + useFallbackOnResolved: () => false, // do NOT use fallback on success as default ...options, }; const retry = retrier(this.logger, options.maxRetries, this.requestPolicy.retryDelay, 10000, true); @@ -95,10 +105,13 @@ export abstract class BaseRestProvider { }); if (statusCode !== 200) { const hostname = new URL(base).hostname; - throw new RequestError(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, statusCode); + throw new RequestError( + `Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, + statusCode, + ); } return { body: body, headers: headers }; - }; + } protected async basePost( base: string, @@ -121,8 +134,11 @@ export abstract class BaseRestProvider { }); if (statusCode !== 200) { const hostname = new URL(base).hostname; - throw new RequestError(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, statusCode); + throw new RequestError( + `Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`, + statusCode, + ); } return { body: body, headers: headers }; - }; + } } diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index 1ea50c1..be75f01 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -3,14 +3,14 @@ import { Inject, Injectable, LoggerService, Optional } from '@nestjs/common'; import { chain } from 'stream-chain'; import { parser } from 'stream-json'; import { connectTo } from 'stream-json/Assembler'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; import { ELBlockSnapshot, ModuleKeys, ModuleKeysFind, Modules, Status } from './response.interface'; import { ConfigService } from '../../config/config.service'; import { PrometheusService, TrackKeysAPIRequest } from '../../prometheus'; import { BaseRestProvider } from '../base/rest-provider'; import { RequestOptions } from '../base/utils/func'; -import BodyReadable from 'undici/types/readable'; -import { IncomingHttpHeaders } from 'undici/types/header'; @Injectable() export class Keysapi extends BaseRestProvider { @@ -46,22 +46,18 @@ export class Keysapi extends BaseRestProvider { } public async getStatus(): Promise { - const { body } = await this.retryRequest( - (baseUrl) => this.baseGet(baseUrl, this.endpoints.status), - ) + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.status)); return (await body.json()) as Status; } public async getModules(): Promise { - const { body } = await this.retryRequest( - (baseUrl) => this.baseGet(baseUrl, this.endpoints.modules), - ); + const { body } = await this.retryRequest((baseUrl) => this.baseGet(baseUrl, this.endpoints.modules)); return (await body.json()) as Modules; } public async getModuleKeys(module_id: string | number, signal?: AbortSignal): Promise { - const resp = await this.retryRequest( - (baseUrl) => this.baseGet(baseUrl, this.endpoints.moduleKeys(module_id), { signal }), + const resp = await this.retryRequest((baseUrl) => + this.baseGet(baseUrl, this.endpoints.moduleKeys(module_id), { signal }), ); // TODO: ignore depositSignature ? const pipeline = chain([resp.body, parser()]); @@ -75,8 +71,8 @@ export class Keysapi extends BaseRestProvider { keysToFind: string[], signal?: AbortSignal, ): Promise { - const { body } = await this.retryRequest( - (baseUrl) => this.basePost(baseUrl, this.endpoints.findModuleKeys(module_id), { pubkeys: keysToFind, signal }), + const { body } = await this.retryRequest((baseUrl) => + this.basePost(baseUrl, this.endpoints.findModuleKeys(module_id), { pubkeys: keysToFind, signal }), ); return (await body.json()) as ModuleKeysFind; } diff --git a/src/daemon/daemon.service.ts b/src/daemon/daemon.service.ts index b46b5b0..8903871 100644 --- a/src/daemon/daemon.service.ts +++ b/src/daemon/daemon.service.ts @@ -1,5 +1,5 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { Inject, Injectable, LoggerService, OnApplicationBootstrap, OnModuleInit } from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnModuleInit } from '@nestjs/common'; import { KeysIndexer } from './services/keys-indexer'; import { RootsProcessor } from './services/roots-processor'; diff --git a/test/cli.e2e-spec.ts b/test/cli.e2e-spec.ts index ae22f01..22a15a7 100644 --- a/test/cli.e2e-spec.ts +++ b/test/cli.e2e-spec.ts @@ -1,5 +1,3 @@ -import { INestApplication } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; import { CommandTestFactory } from 'nest-commander-testing'; import { CliModule } from '../src/cli/cli.module'; @@ -16,10 +14,8 @@ class CustomConfigService extends ConfigService { } describe('Cli (e2e)', () => { - let app: INestApplication; - beforeEach(async () => { - const moduleFixture: TestingModule = await CommandTestFactory.createTestingCommand({ + await CommandTestFactory.createTestingCommand({ imports: [CliModule], }) .overrideProvider(ConfigService)