diff --git a/packages/api/src/beacon/client/index.ts b/packages/api/src/beacon/client/index.ts index 6512d23673c5..6622fe421cd2 100644 --- a/packages/api/src/beacon/client/index.ts +++ b/packages/api/src/beacon/client/index.ts @@ -23,7 +23,7 @@ type ClientModules = HttpClientModules & { httpClient?: IHttpClient; }; -export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods}; +export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods} & {httpClient: IHttpClient}; /** * REST HTTP client for all routes @@ -42,5 +42,6 @@ export function getClient(opts: HttpClientOptions, modules: ClientModules): ApiC node: node.getClient(config, httpClient), proof: proof.getClient(config, httpClient), validator: validator.getClient(config, httpClient), + httpClient, }; } diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index eeccd59d2032..070f6d4fce93 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -50,6 +50,8 @@ export const defaultInit: Required = { export interface IHttpClient { readonly baseUrl: string; + readonly urlsInits: UrlInitRequired[]; + readonly urlsScore: number[]; request( definition: RouteDefinitionExtra, @@ -71,14 +73,13 @@ export type HttpClientModules = { export class HttpClient implements IHttpClient { readonly urlsInits: UrlInitRequired[] = []; + readonly urlsScore: number[]; private readonly signal: null | AbortSignal; private readonly fetch: typeof fetch; private readonly metrics: null | Metrics; private readonly logger: null | Logger; - private readonly urlsScore: number[]; - /** * Cache to keep track of routes per server that do not support SSZ. This cache will only be * populated if we receive a 415 error response from the server after sending a SSZ request body. diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index c686449a29f2..c9a705ae3dc1 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -1,4 +1,5 @@ import tmp from "tmp"; +import {vi} from "vitest"; import type {SecretKey} from "@chainsafe/bls/types"; import {LevelDbController} from "@lodestar/db"; import {interopSecretKey} from "@lodestar/state-transition"; @@ -91,7 +92,7 @@ export async function getAndInitDevValidators({ } export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient { - return mapValues(api, (apiModule) => + const apiClient = mapValues(api, (apiModule) => mapValues(apiModule, (api: (args: unknown, context: unknown) => PromiseLike<{data: unknown; meta: unknown}>) => { return async (args: unknown) => { try { @@ -114,6 +115,15 @@ export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient { }; }) ) as ApiClient; + + apiClient.httpClient = { + baseUrl: "", + request: vi.fn(), + urlsInits: [], + urlsScore: [], + }; + + return apiClient; } export function getNodeApiUrl(node: BeaconNode): string { diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 9cb9f2e2d840..706cf7410b43 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -134,6 +134,21 @@ export class Validator { this.clock.start(this.controller.signal); this.chainHeaderTracker.start(this.controller.signal); + // Add notifier to warn user if primary node is unhealthy as there might + // not be any errors in the logs due to fallback nodes handling the requests + const {httpClient} = this.api; + if (httpClient.urlsInits.length > 1) { + const primaryNodeUrl = toSafePrintableUrl(httpClient.urlsInits[0].baseUrl); + + this.clock?.runEveryEpoch(async () => { + // Only emit warning if URL score is 0 to prevent false positives + // if just a single request fails which might happen due to other reasons + if (httpClient.urlsScore[0] === 0) { + this.logger?.warn("Primary beacon node is unhealthy", {url: primaryNodeUrl}); + } + }); + } + if (metrics) { this.db.setMetrics(metrics.db); diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index 0ee39662952f..ac41c7145128 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -1,7 +1,18 @@ import {vi, Mocked} from "vitest"; -import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode} from "@lodestar/api"; +import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode, IHttpClient} from "@lodestar/api"; -export function getApiClientStub(): {[K in keyof Endpoints]: Mocked>} { +type ApiClientStub = {[K in keyof Endpoints]: Mocked>} & { + httpClient: Mocked; +}; + +const httpClientStub: IHttpClient = { + baseUrl: "", + request: vi.fn(), + urlsInits: [], + urlsScore: [], +}; + +export function getApiClientStub(): ApiClientStub { return { beacon: { getStateValidators: vi.fn(), @@ -25,7 +36,8 @@ export function getApiClientStub(): {[K in keyof Endpoints]: Mocked>}; + httpClient: httpClientStub, + } as unknown as ApiClientStub; } export function mockApiResponse>({