From 1427d34b785efc794d07223171c44e5cd84225ab Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 29 Jun 2024 13:40:51 +0100 Subject: [PATCH 1/5] feat: add warning log if primary beacon node is unhealthy --- packages/api/src/beacon/client/index.ts | 3 ++- packages/api/src/utils/client/httpClient.ts | 5 +++-- packages/validator/src/validator.ts | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/api/src/beacon/client/index.ts b/packages/api/src/beacon/client/index.ts index 6512d23673c5..6622fe421cd2 100644 --- a/packages/api/src/beacon/client/index.ts +++ b/packages/api/src/beacon/client/index.ts @@ -23,7 +23,7 @@ type ClientModules = HttpClientModules & { httpClient?: IHttpClient; }; -export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods}; +export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods} & {httpClient: IHttpClient}; /** * REST HTTP client for all routes @@ -42,5 +42,6 @@ export function getClient(opts: HttpClientOptions, modules: ClientModules): ApiC node: node.getClient(config, httpClient), proof: proof.getClient(config, httpClient), validator: validator.getClient(config, httpClient), + httpClient, }; } diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index eeccd59d2032..070f6d4fce93 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -50,6 +50,8 @@ export const defaultInit: Required = { export interface IHttpClient { readonly baseUrl: string; + readonly urlsInits: UrlInitRequired[]; + readonly urlsScore: number[]; request( definition: RouteDefinitionExtra, @@ -71,14 +73,13 @@ export type HttpClientModules = { export class HttpClient implements IHttpClient { readonly urlsInits: UrlInitRequired[] = []; + readonly urlsScore: number[]; private readonly signal: null | AbortSignal; private readonly fetch: typeof fetch; private readonly metrics: null | Metrics; private readonly logger: null | Logger; - private readonly urlsScore: number[]; - /** * Cache to keep track of routes per server that do not support SSZ. This cache will only be * populated if we receive a 415 error response from the server after sending a SSZ request body. diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 9cb9f2e2d840..1d9e126259db 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -134,6 +134,19 @@ export class Validator { this.clock.start(this.controller.signal); this.chainHeaderTracker.start(this.controller.signal); + // Add notifier to warn user if primary node is unhealthy as there might + // not be any errors in the logs due to fallback nodes handling the requests + const {urlsInits, urlsScore} = this.api.httpClient; + if (urlsInits.length > 1) { + this.clock?.runEveryEpoch(async () => { + // Only emit warning if URL score is 0 to prevent false positives + // if just a single request fails which might happen due to other reasons + if (urlsScore[0] === 0) { + this.logger?.warn("Primary beacon node is unhealthy", {url: toSafePrintableUrl(urlsInits[0].baseUrl)}); + } + }); + } + if (metrics) { this.db.setMetrics(metrics.db); From 3a87b063a542527be880b80aafd36f477cfc6025 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 29 Jun 2024 14:28:29 +0100 Subject: [PATCH 2/5] Fix api client stub type --- packages/validator/test/utils/apiStub.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index 0ee39662952f..6b91a143009a 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -1,7 +1,11 @@ import {vi, Mocked} from "vitest"; -import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode} from "@lodestar/api"; +import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpClient, HttpStatusCode} from "@lodestar/api"; -export function getApiClientStub(): {[K in keyof Endpoints]: Mocked>} { +type ApiClientStub = {[K in keyof Endpoints]: Mocked>} & { + httpClient: Mocked; +}; + +export function getApiClientStub(): ApiClientStub { return { beacon: { getStateValidators: vi.fn(), @@ -25,7 +29,8 @@ export function getApiClientStub(): {[K in keyof Endpoints]: Mocked>}; + httpClient: {}, + } as unknown as ApiClientStub; } export function mockApiResponse>({ From 11946cd3db99fa942dae3af79c9d51f10db777d6 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 29 Jun 2024 15:18:34 +0100 Subject: [PATCH 3/5] Improve http client stub --- packages/validator/test/utils/apiStub.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index 6b91a143009a..ac41c7145128 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -1,8 +1,15 @@ import {vi, Mocked} from "vitest"; -import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpClient, HttpStatusCode} from "@lodestar/api"; +import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode, IHttpClient} from "@lodestar/api"; type ApiClientStub = {[K in keyof Endpoints]: Mocked>} & { - httpClient: Mocked; + httpClient: Mocked; +}; + +const httpClientStub: IHttpClient = { + baseUrl: "", + request: vi.fn(), + urlsInits: [], + urlsScore: [], }; export function getApiClientStub(): ApiClientStub { @@ -29,7 +36,7 @@ export function getApiClientStub(): ApiClientStub { publishAggregateAndProofs: vi.fn(), submitBeaconCommitteeSelections: vi.fn(), }, - httpClient: {}, + httpClient: httpClientStub, } as unknown as ApiClientStub; } From 4074da338f8b707744ee8bde3d53e9e3a2942980 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 29 Jun 2024 15:45:19 +0100 Subject: [PATCH 4/5] Update all api client stubs --- packages/beacon-node/test/utils/node/validator.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index c686449a29f2..c9a705ae3dc1 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -1,4 +1,5 @@ import tmp from "tmp"; +import {vi} from "vitest"; import type {SecretKey} from "@chainsafe/bls/types"; import {LevelDbController} from "@lodestar/db"; import {interopSecretKey} from "@lodestar/state-transition"; @@ -91,7 +92,7 @@ export async function getAndInitDevValidators({ } export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient { - return mapValues(api, (apiModule) => + const apiClient = mapValues(api, (apiModule) => mapValues(apiModule, (api: (args: unknown, context: unknown) => PromiseLike<{data: unknown; meta: unknown}>) => { return async (args: unknown) => { try { @@ -114,6 +115,15 @@ export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient { }; }) ) as ApiClient; + + apiClient.httpClient = { + baseUrl: "", + request: vi.fn(), + urlsInits: [], + urlsScore: [], + }; + + return apiClient; } export function getNodeApiUrl(node: BeaconNode): string { From b42eb4eec3a09d3d317e42d39a3bf8ba263cc54e Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sun, 30 Jun 2024 12:33:44 +0100 Subject: [PATCH 5/5] Only sanitize primary node url once --- packages/validator/src/validator.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 1d9e126259db..706cf7410b43 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -136,13 +136,15 @@ export class Validator { // Add notifier to warn user if primary node is unhealthy as there might // not be any errors in the logs due to fallback nodes handling the requests - const {urlsInits, urlsScore} = this.api.httpClient; - if (urlsInits.length > 1) { + const {httpClient} = this.api; + if (httpClient.urlsInits.length > 1) { + const primaryNodeUrl = toSafePrintableUrl(httpClient.urlsInits[0].baseUrl); + this.clock?.runEveryEpoch(async () => { // Only emit warning if URL score is 0 to prevent false positives // if just a single request fails which might happen due to other reasons - if (urlsScore[0] === 0) { - this.logger?.warn("Primary beacon node is unhealthy", {url: toSafePrintableUrl(urlsInits[0].baseUrl)}); + if (httpClient.urlsScore[0] === 0) { + this.logger?.warn("Primary beacon node is unhealthy", {url: primaryNodeUrl}); } }); }