From be26d51252591a0b96ef8133f5c8eb90e3b7a585 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:33:39 +0200 Subject: [PATCH] (BEDS-461) implement get user machine metrics --- backend/pkg/api/data_access/data_access.go | 1 + backend/pkg/api/data_access/dummy.go | 19 ++++ .../pkg/api/data_access/machine_metrics.go | 15 +++ backend/pkg/api/handlers/auth.go | 17 ++-- backend/pkg/api/handlers/common.go | 22 ++--- backend/pkg/api/handlers/machine_metrics.go | 37 +++++++ backend/pkg/api/handlers/public.go | 6 +- backend/pkg/api/router.go | 4 +- backend/pkg/api/types/machine_metrics.go | 79 +++++++++++++++ frontend/types/api/machine_metrics.ts | 96 +++++++++++++++++++ 10 files changed, 273 insertions(+), 23 deletions(-) create mode 100644 backend/pkg/api/data_access/machine_metrics.go create mode 100644 backend/pkg/api/handlers/machine_metrics.go create mode 100644 backend/pkg/api/types/machine_metrics.go create mode 100644 frontend/types/api/machine_metrics.ts diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go index 3fc31d105..047d471a6 100644 --- a/backend/pkg/api/data_access/data_access.go +++ b/backend/pkg/api/data_access/data_access.go @@ -30,6 +30,7 @@ type DataAccessor interface { ProtocolRepository RatelimitRepository HealthzRepository + MachineRepository StartDataAccessServices() Close() diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 3e42c6384..39aa31d10 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -6,11 +6,13 @@ import ( "fmt" "math/rand/v2" "reflect" + "slices" "time" "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/options" "github.com/gobitfly/beaconchain/pkg/api/enums" + "github.com/gobitfly/beaconchain/pkg/api/types" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/userservice" "github.com/shopspring/decimal" @@ -656,3 +658,20 @@ func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleV func (d *DummyService) GetValidatorDashboardMobileWidget(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.MobileWidgetData, error) { return getDummyStruct[t.MobileWidgetData]() } + +func (d *DummyService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit uint64, offset uint64) (*types.MachineMetricsData, error) { + data, err := getDummyStruct[types.MachineMetricsData]() + if err != nil { + return nil, err + } + data.SystemMetrics = slices.SortedFunc(slices.Values(data.SystemMetrics), func(i, j *t.MachineMetricSystem) int { + return int(i.Timestamp) - int(j.Timestamp) + }) + data.ValidatorMetrics = slices.SortedFunc(slices.Values(data.ValidatorMetrics), func(i, j *t.MachineMetricValidator) int { + return int(i.Timestamp) - int(j.Timestamp) + }) + data.NodeMetrics = slices.SortedFunc(slices.Values(data.NodeMetrics), func(i, j *t.MachineMetricNode) int { + return int(i.Timestamp) - int(j.Timestamp) + }) + return data, nil +} diff --git a/backend/pkg/api/data_access/machine_metrics.go b/backend/pkg/api/data_access/machine_metrics.go new file mode 100644 index 000000000..3b9935fcf --- /dev/null +++ b/backend/pkg/api/data_access/machine_metrics.go @@ -0,0 +1,15 @@ +package dataaccess + +import ( + "context" + + "github.com/gobitfly/beaconchain/pkg/api/types" +) + +type MachineRepository interface { + GetUserMachineMetrics(context context.Context, userID uint64, limit uint64, offset uint64) (*types.MachineMetricsData, error) +} + +func (d *DataAccessService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit uint64, offset uint64) (*types.MachineMetricsData, error) { + return d.dummy.GetUserMachineMetrics(ctx, userID, limit, offset) +} diff --git a/backend/pkg/api/handlers/auth.go b/backend/pkg/api/handlers/auth.go index 598b8f713..ee30cedda 100644 --- a/backend/pkg/api/handlers/auth.go +++ b/backend/pkg/api/handlers/auth.go @@ -1,6 +1,7 @@ package handlers import ( + "cmp" "context" "errors" "fmt" @@ -181,13 +182,15 @@ const authHeaderPrefix = "Bearer " func (h *HandlerService) GetUserIdByApiKey(r *http.Request) (uint64, error) { // TODO: store user id in context during ratelimting and use it here - var apiKey string - authHeader := r.Header.Get("Authorization") - if strings.HasPrefix(authHeader, authHeaderPrefix) { - apiKey = strings.TrimPrefix(authHeader, authHeaderPrefix) - } else { - apiKey = r.URL.Query().Get("api_key") - } + query := r.URL.Query() + header := r.Header + apiKey := cmp.Or( + strings.TrimPrefix(header.Get("Authorization"), authHeaderPrefix), + header.Get("X-Api-Key"), + query.Get("api_key"), + query.Get("apiKey"), + query.Get("apikey"), + ) if apiKey == "" { return 0, newUnauthorizedErr("missing api key") } diff --git a/backend/pkg/api/handlers/common.go b/backend/pkg/api/handlers/common.go index a667b029b..1af4a9234 100644 --- a/backend/pkg/api/handlers/common.go +++ b/backend/pkg/api/handlers/common.go @@ -29,11 +29,12 @@ import ( ) type HandlerService struct { - dai dataaccess.DataAccessor - scs *scs.SessionManager + dai dataaccess.DataAccessor + scs *scs.SessionManager + isPostMachineMetricsEnabled bool // if more config options are needed, consider having the whole config in here } -func NewHandlerService(dataAccessor dataaccess.DataAccessor, sessionManager *scs.SessionManager) *HandlerService { +func NewHandlerService(dataAccessor dataaccess.DataAccessor, sessionManager *scs.SessionManager, enablePostMachineMetrics bool) *HandlerService { if allNetworks == nil { networks, err := dataAccessor.GetAllNetworks() if err != nil { @@ -43,8 +44,9 @@ func NewHandlerService(dataAccessor dataaccess.DataAccessor, sessionManager *scs } return &HandlerService{ - dai: dataAccessor, - scs: sessionManager, + dai: dataAccessor, + scs: sessionManager, + isPostMachineMetricsEnabled: enablePostMachineMetrics, } } @@ -525,14 +527,10 @@ func checkEnum[T enums.EnumFactory[T]](v *validationError, enumString string, na return enum } -// checkEnumIsAllowed checks if the given enum is in the list of allowed enums. -func checkEnumIsAllowed[T enums.EnumFactory[T]](v *validationError, enum T, allowed []T, name string) { - if enums.IsInvalidEnum(enum) { - v.add(name, "parameter is missing or invalid, please check the API documentation") - return - } +// better func name would be +func checkValueInAllowed[T cmp.Ordered](v *validationError, value T, allowed []T, name string) { for _, a := range allowed { - if enum.Int() == a.Int() { + if cmp.Compare(value, a) == 0 { return } } diff --git a/backend/pkg/api/handlers/machine_metrics.go b/backend/pkg/api/handlers/machine_metrics.go new file mode 100644 index 000000000..5a84ce435 --- /dev/null +++ b/backend/pkg/api/handlers/machine_metrics.go @@ -0,0 +1,37 @@ +package handlers + +import ( + "net/http" + + "github.com/gobitfly/beaconchain/pkg/api/types" +) + +func (h *HandlerService) InternalGetUserMachineMetrics(w http.ResponseWriter, r *http.Request) { + h.PublicGetUserMachineMetrics(w, r) +} + +func (h *HandlerService) PublicGetUserMachineMetrics(w http.ResponseWriter, r *http.Request) { + var v validationError + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, r, err) + return + } + q := r.URL.Query() + offset := v.checkUint(q.Get("offset"), "offset") + limit := uint64(180) + if limitParam := q.Get("limit"); limitParam != "" { + limit = v.checkUint(limitParam, "limit") + } + + data, err := h.dai.GetUserMachineMetrics(r.Context(), userId, limit, offset) + if err != nil { + handleErr(w, r, err) + return + } + response := types.GetUserMachineMetricsRespone{ + Data: *data, + } + + returnOk(w, r, response) +} diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index a2a92171a..3e5b2e2cc 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -1018,7 +1018,7 @@ func (h *HandlerService) PublicGetValidatorDashboardSummary(w http.ResponseWrite period := checkEnum[enums.TimePeriod](&v, q.Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h - checkEnumIsAllowed(&v, period, summaryAllowedPeriods, "period") + checkValueInAllowed(&v, period, summaryAllowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return @@ -1065,7 +1065,7 @@ func (h *HandlerService) PublicGetValidatorDashboardGroupSummary(w http.Response groupId := v.checkGroupId(vars["group_id"], forbidEmpty) period := checkEnum[enums.TimePeriod](&v, r.URL.Query().Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h - checkEnumIsAllowed(&v, period, summaryAllowedPeriods, "period") + checkValueInAllowed(&v, period, summaryAllowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return @@ -1160,7 +1160,7 @@ func (h *HandlerService) PublicGetValidatorDashboardSummaryValidators(w http.Res period := checkEnum[enums.TimePeriod](&v, q.Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h allowedPeriods := []enums.TimePeriod{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} - checkEnumIsAllowed(&v, period, allowedPeriods, "period") + checkValueInAllowed(&v, period, allowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 16ab4c0b2..c9202848a 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -32,7 +32,7 @@ func NewApiRouter(dataAccessor dataaccess.DataAccessor, cfg *types.Config) *mux. if !(cfg.Frontend.CsrfInsecure || cfg.Frontend.Debug) { internalRouter.Use(getCsrfProtectionMiddleware(cfg), csrfInjecterMiddleware) } - handlerService := handlers.NewHandlerService(dataAccessor, sessionManager) + handlerService := handlers.NewHandlerService(dataAccessor, sessionManager, !cfg.Frontend.DisableStatsInserts) // store user id in context, if available publicRouter.Use(handlers.GetUserIdStoreMiddleware(handlerService.GetUserIdByApiKey)) @@ -123,6 +123,8 @@ func addRoutes(hs *handlers.HandlerService, publicRouter, internalRouter *mux.Ro {http.MethodGet, "/users/me/dashboards", hs.PublicGetUserDashboards, hs.InternalGetUserDashboards}, {http.MethodPut, "/users/me/notifications/settings/paired-devices/{client_id}/token", nil, hs.InternalPostUsersMeNotificationSettingsPairedDevicesToken}, + {http.MethodGet, "/users/me/machine-metrics", hs.PublicGetUserMachineMetrics, hs.InternalGetUserMachineMetrics}, + {http.MethodPost, "/search", nil, hs.InternalPostSearch}, {http.MethodPost, "/account-dashboards", hs.PublicPostAccountDashboards, hs.InternalPostAccountDashboards}, diff --git a/backend/pkg/api/types/machine_metrics.go b/backend/pkg/api/types/machine_metrics.go new file mode 100644 index 000000000..059c06471 --- /dev/null +++ b/backend/pkg/api/types/machine_metrics.go @@ -0,0 +1,79 @@ +package types + +type MachineMetricSystem struct { + Timestamp uint64 `json:"timestamp,omitempty" faker:"boundary_start=1725166800, boundary_end=1725177600"` + ExporterVersion string `json:"exporter_version,omitempty"` + // system + CpuCores uint64 `json:"cpu_cores,omitempty"` + CpuThreads uint64 `json:"cpu_threads,omitempty"` + CpuNodeSystemSecondsTotal uint64 `json:"cpu_node_system_seconds_total,omitempty"` + CpuNodeUserSecondsTotal uint64 `json:"cpu_node_user_seconds_total,omitempty"` + CpuNodeIowaitSecondsTotal uint64 `json:"cpu_node_iowait_seconds_total,omitempty"` + CpuNodeIdleSecondsTotal uint64 `json:"cpu_node_idle_seconds_total,omitempty"` + MemoryNodeBytesTotal uint64 `json:"memory_node_bytes_total,omitempty"` + MemoryNodeBytesFree uint64 `json:"memory_node_bytes_free,omitempty"` + MemoryNodeBytesCached uint64 `json:"memory_node_bytes_cached,omitempty"` + MemoryNodeBytesBuffers uint64 `json:"memory_node_bytes_buffers,omitempty"` + DiskNodeBytesTotal uint64 `json:"disk_node_bytes_total,omitempty"` + DiskNodeBytesFree uint64 `json:"disk_node_bytes_free,omitempty"` + DiskNodeIoSeconds uint64 `json:"disk_node_io_seconds,omitempty"` + DiskNodeReadsTotal uint64 `json:"disk_node_reads_total,omitempty"` + DiskNodeWritesTotal uint64 `json:"disk_node_writes_total,omitempty"` + NetworkNodeBytesTotalReceive uint64 `json:"network_node_bytes_total_receive,omitempty"` + NetworkNodeBytesTotalTransmit uint64 `json:"network_node_bytes_total_transmit,omitempty"` + MiscNodeBootTsSeconds uint64 `json:"misc_node_boot_ts_seconds,omitempty"` + MiscOs string `json:"misc_os,omitempty"` + // do not store in bigtable but include them in generated model + Machine *string `json:"machine,omitempty"` +} + +type MachineMetricValidator struct { + Timestamp uint64 `json:"timestamp,omitempty" faker:"boundary_start=1725166800, boundary_end=1725177600"` + ExporterVersion string `json:"exporter_version,omitempty"` + // process + CpuProcessSecondsTotal uint64 `json:"cpu_process_seconds_total,omitempty"` + MemoryProcessBytes uint64 `json:"memory_process_bytes,omitempty"` + ClientName string `json:"client_name,omitempty"` + ClientVersion string `json:"client_version,omitempty"` + ClientBuild uint64 `json:"client_build,omitempty"` + SyncEth2FallbackConfigured bool `json:"sync_eth2_fallback_configured,omitempty"` + SyncEth2FallbackConnected bool `json:"sync_eth2_fallback_connected,omitempty"` + // validator + ValidatorTotal uint64 `json:"validator_total,omitempty"` + ValidatorActive uint64 `json:"validator_active,omitempty"` + // do not store in bigtable but include them in generated model + Machine *string `json:"machine,omitempty"` +} + +type MachineMetricNode struct { + Timestamp uint64 `json:"timestamp,omitempty" faker:"boundary_start=1725166800, boundary_end=1725177600"` + ExporterVersion string `json:"exporter_version,omitempty"` + // process + CpuProcessSecondsTotal uint64 `json:"cpu_process_seconds_total,omitempty"` + MemoryProcessBytes uint64 `json:"memory_process_bytes,omitempty"` + ClientName string `json:"client_name,omitempty"` + ClientVersion string `json:"client_version,omitempty"` + ClientBuild uint64 `json:"client_build,omitempty"` + SyncEth2FallbackConfigured bool `json:"sync_eth2_fallback_configured,omitempty"` + SyncEth2FallbackConnected bool `json:"sync_eth2_fallback_connected,omitempty"` + // node + DiskBeaconchainBytesTotal uint64 `json:"disk_beaconchain_bytes_total,omitempty"` + NetworkLibp2PBytesTotalReceive uint64 `json:"network_libp2p_bytes_total_receive,omitempty"` + NetworkLibp2PBytesTotalTransmit uint64 `json:"network_libp2p_bytes_total_transmit,omitempty"` + NetworkPeersConnected uint64 `json:"network_peers_connected,omitempty"` + SyncEth1Connected bool `json:"sync_eth1_connected,omitempty"` + SyncEth2Synced bool `json:"sync_eth2_synced,omitempty"` + SyncBeaconHeadSlot uint64 `json:"sync_beacon_head_slot,omitempty"` + SyncEth1FallbackConfigured bool `json:"sync_eth1_fallback_configured,omitempty"` + SyncEth1FallbackConnected bool `json:"sync_eth1_fallback_connected,omitempty"` + // do not store in bigtable but include them in generated model + Machine *string `json:"machine,omitempty"` +} + +type MachineMetricsData struct { + SystemMetrics []*MachineMetricSystem `json:"system_metrics" faker:"slice_len=30"` + ValidatorMetrics []*MachineMetricValidator `json:"validator_metrics" faker:"slice_len=30"` + NodeMetrics []*MachineMetricNode `json:"node_metrics" faker:"slice_len=30"` +} + +type GetUserMachineMetricsRespone ApiDataResponse[MachineMetricsData] diff --git a/frontend/types/api/machine_metrics.ts b/frontend/types/api/machine_metrics.ts new file mode 100644 index 000000000..78c773286 --- /dev/null +++ b/frontend/types/api/machine_metrics.ts @@ -0,0 +1,96 @@ +// Code generated by tygo. DO NOT EDIT. +/* eslint-disable */ +import type { ApiDataResponse } from './common' + +////////// +// source: machine_metrics.go + +export interface MachineMetricSystem { + timestamp?: number /* uint64 */; + exporter_version?: string; + /** + * system + */ + cpu_cores?: number /* uint64 */; + cpu_threads?: number /* uint64 */; + cpu_node_system_seconds_total?: number /* uint64 */; + cpu_node_user_seconds_total?: number /* uint64 */; + cpu_node_iowait_seconds_total?: number /* uint64 */; + cpu_node_idle_seconds_total?: number /* uint64 */; + memory_node_bytes_total?: number /* uint64 */; + memory_node_bytes_free?: number /* uint64 */; + memory_node_bytes_cached?: number /* uint64 */; + memory_node_bytes_buffers?: number /* uint64 */; + disk_node_bytes_total?: number /* uint64 */; + disk_node_bytes_free?: number /* uint64 */; + disk_node_io_seconds?: number /* uint64 */; + disk_node_reads_total?: number /* uint64 */; + disk_node_writes_total?: number /* uint64 */; + network_node_bytes_total_receive?: number /* uint64 */; + network_node_bytes_total_transmit?: number /* uint64 */; + misc_node_boot_ts_seconds?: number /* uint64 */; + misc_os?: string; + /** + * do not store in bigtable but include them in generated model + */ + machine?: string; +} +export interface MachineMetricValidator { + timestamp?: number /* uint64 */; + exporter_version?: string; + /** + * process + */ + cpu_process_seconds_total?: number /* uint64 */; + memory_process_bytes?: number /* uint64 */; + client_name?: string; + client_version?: string; + client_build?: number /* uint64 */; + sync_eth2_fallback_configured?: boolean; + sync_eth2_fallback_connected?: boolean; + /** + * validator + */ + validator_total?: number /* uint64 */; + validator_active?: number /* uint64 */; + /** + * do not store in bigtable but include them in generated model + */ + machine?: string; +} +export interface MachineMetricNode { + timestamp?: number /* uint64 */; + exporter_version?: string; + /** + * process + */ + cpu_process_seconds_total?: number /* uint64 */; + memory_process_bytes?: number /* uint64 */; + client_name?: string; + client_version?: string; + client_build?: number /* uint64 */; + sync_eth2_fallback_configured?: boolean; + sync_eth2_fallback_connected?: boolean; + /** + * node + */ + disk_beaconchain_bytes_total?: number /* uint64 */; + network_libp2p_bytes_total_receive?: number /* uint64 */; + network_libp2p_bytes_total_transmit?: number /* uint64 */; + network_peers_connected?: number /* uint64 */; + sync_eth1_connected?: boolean; + sync_eth2_synced?: boolean; + sync_beacon_head_slot?: number /* uint64 */; + sync_eth1_fallback_configured?: boolean; + sync_eth1_fallback_connected?: boolean; + /** + * do not store in bigtable but include them in generated model + */ + machine?: string; +} +export interface MachineMetricsData { + system_metrics: (MachineMetricSystem | undefined)[]; + validator_metrics: (MachineMetricValidator | undefined)[]; + node_metrics: (MachineMetricNode | undefined)[]; +} +export type GetUserMachineMetricsRespone = ApiDataResponse;