From 3338f729855a4e6aa91a244bf813ee60c8a6f632 Mon Sep 17 00:00:00 2001
From: moshe-blox <moshe@blox.io>
Date: Sun, 10 Sep 2023 17:13:36 +0300
Subject: [PATCH 01/34] chore: link to SSV API docs in configs & README

---
 README.md                           | 1 +
 config/config.example.yaml          | 6 +++++-
 config/config.exporter.example.yaml | 4 ++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c5446cc838..ef7021d98a 100644
--- a/README.md
+++ b/README.md
@@ -47,6 +47,7 @@ to run validators in a decentralized and trustless way.
 The following documents contain instructions and information on how to get started:
 * [Operator Node Installation](https://docs.ssv.network/run-a-node/operator-node/installation)
 * [Developers' Guide](./docs/DEV_GUIDE.md)
+* [SSV API Docs](https://bloxapp.github.io/ssv/)
 
 ## Contribution
 
diff --git a/config/config.example.yaml b/config/config.example.yaml
index 92779ee019..04e65a3a91 100644
--- a/config/config.example.yaml
+++ b/config/config.example.yaml
@@ -39,4 +39,8 @@ p2p:
 OperatorPrivateKey:
 
 # This enables monitoring at the specified port, see https://github.com/bloxapp/ssv/tree/main/monitoring
-MetricsAPIPort: 15000
\ No newline at end of file
+MetricsAPIPort: 15000
+
+# This enables the SSV API at the specified port. Refer to the documentation at https://bloxapp.github.io/ssv/
+# It's recommended to keep this port private to prevent potential resource-intensive attacks.
+# SSVAPIPort: 16000
\ No newline at end of file
diff --git a/config/config.exporter.example.yaml b/config/config.exporter.example.yaml
index f8c8402480..9953ad453a 100644
--- a/config/config.exporter.example.yaml
+++ b/config/config.exporter.example.yaml
@@ -35,3 +35,7 @@ bootnode:
 
 LocalEventsPath: # path to local events. used for running the node with custom local events
 WebSocketAPIPort: 16000
+
+# This enables the SSV API at the specified port. Refer to the documentation at https://bloxapp.github.io/ssv/
+# It's recommended to keep this port private to prevent potential resource-intensive attacks.
+# SSVAPIPort: 16000
\ No newline at end of file

From 6e9f9ebdcab200ef57108885cdd5951d93e45cfc Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 10 Nov 2023 19:00:30 +0600
Subject: [PATCH 02/34] initial commit

---
 api/handlers/node.go | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 323ae0642c..a7a9c6da67 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -3,10 +3,11 @@ package handlers
 import (
 	"net/http"
 
-	"github.com/bloxapp/ssv/api"
-	networkpeers "github.com/bloxapp/ssv/network/peers"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
+
+	"github.com/bloxapp/ssv/api"
+	networkpeers "github.com/bloxapp/ssv/network/peers"
 )
 
 type TopicIndex interface {
@@ -44,6 +45,11 @@ type identityJSON struct {
 	Version   string   `json:"version"`
 }
 
+type healthCheckJSON struct {
+	PeersConnectionHealth  string `json:"peers_connection_health"`
+	BeaconConnectionHealth string `json:"beacon_connection_health"`
+}
+
 type Node struct {
 	PeersIndex networkpeers.Index
 	TopicIndex TopicIndex
@@ -106,3 +112,16 @@ func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
 
 	return api.Render(w, r, alland)
 }
+
+func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
+	resp := healthCheckJSON{}
+	switch l := len(h.Network.Peers()); {
+	case l == 0:
+		resp.PeersConnectionHealth = "red"
+	case l > 0 && l <= 10:
+		resp.PeersConnectionHealth = "yellow"
+	case l > 10:
+		resp.PeersConnectionHealth = "green"
+	}
+	return api.Render(w, r, resp)
+}

From 0da513ed476636cd8e7052ce711354c13cf920db Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 13 Nov 2023 18:09:01 +0600
Subject: [PATCH 03/34] add node health route to ssv API

---
 api/handlers/node.go   | 87 +++++++++++++++++++++++++++---------------
 api/server/server.go   |  6 ++-
 cli/operator/node.go   |  3 +-
 nodeprobe/nodeprobe.go | 24 ++++++++++++
 4 files changed, 87 insertions(+), 33 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index a7a9c6da67..bda320b07c 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"context"
 	"net/http"
 
 	"github.com/libp2p/go-libp2p/core/network"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/bloxapp/ssv/api"
 	networkpeers "github.com/bloxapp/ssv/network/peers"
+	"github.com/bloxapp/ssv/nodeprobe"
 )
 
 type TopicIndex interface {
@@ -46,14 +48,18 @@ type identityJSON struct {
 }
 
 type healthCheckJSON struct {
-	PeersConnectionHealth  string `json:"peers_connection_health"`
-	BeaconConnectionHealth string `json:"beacon_connection_health"`
+	Peers               []peerJSON `json:"peers"`
+	BeaconConnected     bool       `json:"beacon_connected"`
+	ExecutionConnected  bool       `json:"execution_connected"`
+	EventSyncConnected  bool       `json:"event_sync_connected"`
+	LocalPortsListening []string   `json:"local_ports_listening"`
 }
 
 type Node struct {
 	PeersIndex networkpeers.Index
 	TopicIndex TopicIndex
 	Network    network.Network
+	NodeProber *nodeprobe.Prober
 }
 
 func (h *Node) Identity(w http.ResponseWriter, r *http.Request) error {
@@ -70,7 +76,54 @@ func (h *Node) Identity(w http.ResponseWriter, r *http.Request) error {
 }
 
 func (h *Node) Peers(w http.ResponseWriter, r *http.Request) error {
-	peers := h.Network.Peers()
+	prs := h.Network.Peers()
+	resp := h.peers(prs)
+	return api.Render(w, r, resp)
+}
+
+func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
+	allpeers, peerbytpc := h.TopicIndex.PeersByTopic()
+	alland := AllPeersAndTopicsJSON{}
+	tpcs := []topicIndexJSON{}
+	for topic, peerz := range peerbytpc {
+		tpcs = append(tpcs, topicIndexJSON{TopicName: topic, Peers: peerz})
+	}
+	alland.AllPeers = allpeers
+	alland.PeersByTopic = tpcs
+
+	return api.Render(w, r, alland)
+}
+
+func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
+	ctx := context.Background()
+	resp := healthCheckJSON{}
+	// check ports being used
+	addrs := h.Network.ListenAddresses()
+	for _, addr := range addrs {
+		resp.LocalPortsListening = append(resp.LocalPortsListening, addr.String())
+	}
+	// check consensus node health
+	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
+	if err == nil {
+		resp.BeaconConnected = true
+	}
+	// check execution node health
+	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
+	if err == nil {
+		resp.EventSyncConnected = true
+	}
+	// check event sync health
+	err = h.NodeProber.CheckEventSycNodeHealth(ctx)
+	if err != nil {
+		resp.EventSyncConnected = true
+	}
+	// check peers connection
+	prs := h.Network.Peers()
+	resp.Peers = h.peers(prs)
+	return api.Render(w, r, resp)
+}
+
+func (h *Node) peers(peers []peer.ID) []peerJSON {
 	resp := make([]peerJSON, len(peers))
 	for i, id := range peers {
 		resp[i] = peerJSON{
@@ -97,31 +150,5 @@ func (h *Node) Peers(w http.ResponseWriter, r *http.Request) error {
 		}
 		resp[i].Version = nodeInfo.Metadata.NodeVersion
 	}
-	return api.Render(w, r, resp)
-}
-
-func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
-	allpeers, peerbytpc := h.TopicIndex.PeersByTopic()
-	alland := AllPeersAndTopicsJSON{}
-	tpcs := []topicIndexJSON{}
-	for topic, peerz := range peerbytpc {
-		tpcs = append(tpcs, topicIndexJSON{TopicName: topic, Peers: peerz})
-	}
-	alland.AllPeers = allpeers
-	alland.PeersByTopic = tpcs
-
-	return api.Render(w, r, alland)
-}
-
-func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
-	resp := healthCheckJSON{}
-	switch l := len(h.Network.Peers()); {
-	case l == 0:
-		resp.PeersConnectionHealth = "red"
-	case l > 0 && l <= 10:
-		resp.PeersConnectionHealth = "yellow"
-	case l > 10:
-		resp.PeersConnectionHealth = "green"
-	}
-	return api.Render(w, r, resp)
+	return resp
 }
diff --git a/api/server/server.go b/api/server/server.go
index 26b0d7cddb..63e9b2c4bd 100644
--- a/api/server/server.go
+++ b/api/server/server.go
@@ -5,11 +5,12 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/bloxapp/ssv/api"
-	"github.com/bloxapp/ssv/api/handlers"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"go.uber.org/zap"
+
+	"github.com/bloxapp/ssv/api"
+	"github.com/bloxapp/ssv/api/handlers"
 )
 
 type Server struct {
@@ -45,6 +46,7 @@ func (s *Server) Run() error {
 	router.Get("/v1/node/peers", api.Handler(s.node.Peers))
 	router.Get("/v1/node/topics", api.Handler(s.node.Topics))
 	router.Get("/v1/validators", api.Handler(s.validators.List))
+	router.Get("/v1/node/health", api.Handler(s.node.Health))
 
 	s.logger.Info("Serving SSV API", zap.String("addr", s.addr))
 
diff --git a/cli/operator/node.go b/cli/operator/node.go
index a6bca602c3..44f6a6aea5 100644
--- a/cli/operator/node.go
+++ b/cli/operator/node.go
@@ -12,13 +12,13 @@ import (
 	"os"
 	"time"
 
-	spectypes "github.com/bloxapp/ssv-spec/types"
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ilyakaznacheev/cleanenv"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
+	spectypes "github.com/bloxapp/ssv-spec/types"
 	"github.com/bloxapp/ssv/api/handlers"
 	apiserver "github.com/bloxapp/ssv/api/server"
 	"github.com/bloxapp/ssv/beacon/goclient"
@@ -295,6 +295,7 @@ var StartNodeCmd = &cobra.Command{
 					PeersIndex: p2pNetwork.(p2pv1.PeersIndexProvider).PeersIndex(),
 					Network:    p2pNetwork.(p2pv1.HostProvider).Host().Network(),
 					TopicIndex: p2pNetwork.(handlers.TopicIndex),
+					NodeProber: nodeProber,
 				},
 				&handlers.Validators{
 					Shares: nodeStorage.Shares(),
diff --git a/nodeprobe/nodeprobe.go b/nodeprobe/nodeprobe.go
index d62c004afc..d24c7d858c 100644
--- a/nodeprobe/nodeprobe.go
+++ b/nodeprobe/nodeprobe.go
@@ -136,3 +136,27 @@ func (p *Prober) AddNode(name string, node Node) {
 
 	p.nodes[name] = node
 }
+
+func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
+	err := p.nodes["consensus client"].Healthy(ctx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *Prober) CheckExecutionNodeHealth(ctx context.Context) error {
+	err := p.nodes["execution client"].Healthy(ctx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *Prober) CheckEventSycNodeHealth(ctx context.Context) error {
+	err := p.nodes["event syncer"].Healthy(ctx)
+	if err != nil {
+		return err
+	}
+	return nil
+}

From 1a66ceee185ae04b7fd3fe9a463ad10d1e56ad1b Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 13 Nov 2023 18:40:31 +0600
Subject: [PATCH 04/34] update health route

---
 api/handlers/node.go | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index bda320b07c..c66facdb58 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -48,11 +48,11 @@ type identityJSON struct {
 }
 
 type healthCheckJSON struct {
-	Peers               []peerJSON `json:"peers"`
-	BeaconConnected     bool       `json:"beacon_connected"`
-	ExecutionConnected  bool       `json:"execution_connected"`
-	EventSyncConnected  bool       `json:"event_sync_connected"`
-	LocalPortsListening []string   `json:"local_ports_listening"`
+	PeersHealthStatus               string   `json:"peers_status"`
+	BeaconConnectionHealthStatus    string   `json:"beacon_health_status"`
+	ExecutionConnectionHealthStatus string   `json:"execution_health_status"`
+	EventSyncHealthStatus           string   `json:"event_sync_health_status"`
+	LocalPortsListening             []string `json:"local_ports_listening"`
 }
 
 type Node struct {
@@ -104,22 +104,38 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
-	if err == nil {
-		resp.BeaconConnected = true
+	if err != nil {
+		resp.BeaconConnectionHealthStatus = "bad"
+	} else {
+		resp.BeaconConnectionHealthStatus = "good"
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
-	if err == nil {
-		resp.EventSyncConnected = true
+	if err != nil {
+		resp.ExecutionConnectionHealthStatus = "bad"
+	} else {
+		resp.ExecutionConnectionHealthStatus = "good"
 	}
 	// check event sync health
 	err = h.NodeProber.CheckEventSycNodeHealth(ctx)
 	if err != nil {
-		resp.EventSyncConnected = true
+		resp.EventSyncHealthStatus = "bad"
+	} else {
+		resp.EventSyncHealthStatus = "good"
 	}
 	// check peers connection
+	var activePeerCount int
 	prs := h.Network.Peers()
-	resp.Peers = h.peers(prs)
+	for _, p := range h.peers(prs) {
+		if p.Connectedness == "Connected" {
+			activePeerCount++
+		}
+	}
+	if activePeerCount < 10 {
+		resp.PeersHealthStatus = "bad"
+	} else {
+		resp.PeersHealthStatus = "good"
+	}
 	return api.Render(w, r, resp)
 }
 

From f21b1aed2c713113f632f05b9f8b8009e8383f11 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 13 Nov 2023 18:56:24 +0600
Subject: [PATCH 05/34] update health route

---
 api/handlers/node.go | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index c66facdb58..46525b25f5 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -2,6 +2,7 @@ package handlers
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/libp2p/go-libp2p/core/network"
@@ -16,6 +17,21 @@ type TopicIndex interface {
 	PeersByTopic() ([]peer.ID, map[string][]peer.ID)
 }
 
+type HealthStatus int
+
+const (
+	NotHealthy HealthStatus = iota
+	Healthy
+)
+
+func (c HealthStatus) String() string {
+	str := [...]string{"NotHealthy", "Healthy"}
+	if c < 0 || int(c) >= len(str) {
+		return "(unrecognized)"
+	}
+	return str[c]
+}
+
 type AllPeersAndTopicsJSON struct {
 	AllPeers     []peer.ID        `json:"all_peers"`
 	PeersByTopic []topicIndexJSON `json:"peers_by_topic"`
@@ -104,24 +120,24 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
-	if err != nil {
-		resp.BeaconConnectionHealthStatus = "bad"
+	if err == nil {
+		resp.BeaconConnectionHealthStatus = Healthy.String()
 	} else {
-		resp.BeaconConnectionHealthStatus = "good"
+		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
-	if err != nil {
-		resp.ExecutionConnectionHealthStatus = "bad"
+	if err == nil {
+		resp.ExecutionConnectionHealthStatus = Healthy.String()
 	} else {
-		resp.ExecutionConnectionHealthStatus = "good"
+		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check event sync health
 	err = h.NodeProber.CheckEventSycNodeHealth(ctx)
 	if err != nil {
-		resp.EventSyncHealthStatus = "bad"
+		resp.EventSyncHealthStatus = Healthy.String()
 	} else {
-		resp.EventSyncHealthStatus = "good"
+		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check peers connection
 	var activePeerCount int
@@ -131,10 +147,10 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 			activePeerCount++
 		}
 	}
-	if activePeerCount < 10 {
-		resp.PeersHealthStatus = "bad"
+	if activePeerCount >= 10 {
+		resp.PeersHealthStatus = Healthy.String()
 	} else {
-		resp.PeersHealthStatus = "good"
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 10 peers are connected")
 	}
 	return api.Render(w, r, resp)
 }

From a7cbc3f916ee10c3c492d09561c4ff58c8fae1b8 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 13 Nov 2023 19:04:45 +0600
Subject: [PATCH 06/34] update health route

---
 api/handlers/node.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 46525b25f5..d1e063a4a7 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -147,10 +147,13 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 			activePeerCount++
 		}
 	}
-	if activePeerCount >= 10 {
-		resp.PeersHealthStatus = Healthy.String()
-	} else {
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 10 peers are connected")
+	switch count := activePeerCount; {
+	case count >= 5:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %d  peers are connected", Healthy.String(), activePeerCount)
+	case count < 5:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 5 peers are connected")
+	case count == 0:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "error: no peers are connected")
 	}
 	return api.Render(w, r, resp)
 }

From 0aee6491782cf45cb5e6e12e9b7c5787175b8bae Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 14 Nov 2023 14:27:18 +0600
Subject: [PATCH 07/34] update health route

---
 api/handlers/node.go   | 24 +++++++++++-------------
 nodeprobe/nodeprobe.go |  8 +++++++-
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index d1e063a4a7..69e466d643 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -112,7 +112,11 @@ func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
 
 func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	ctx := context.Background()
-	resp := healthCheckJSON{}
+	resp := healthCheckJSON{
+		BeaconConnectionHealthStatus:    Healthy.String(),
+		ExecutionConnectionHealthStatus: Healthy.String(),
+		EventSyncHealthStatus:           Healthy.String(),
+	}
 	// check ports being used
 	addrs := h.Network.ListenAddresses()
 	for _, addr := range addrs {
@@ -120,23 +124,17 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
-	if err == nil {
-		resp.BeaconConnectionHealthStatus = Healthy.String()
-	} else {
+	if err != nil {
 		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
-	if err == nil {
-		resp.ExecutionConnectionHealthStatus = Healthy.String()
-	} else {
+	if err != nil {
 		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check event sync health
-	err = h.NodeProber.CheckEventSycNodeHealth(ctx)
+	err = h.NodeProber.CheckEventSyncerHealth(ctx)
 	if err != nil {
-		resp.EventSyncHealthStatus = Healthy.String()
-	} else {
 		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
 	}
 	// check peers connection
@@ -148,10 +146,10 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 	switch count := activePeerCount; {
-	case count >= 5:
+	case count >= 10:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: %d  peers are connected", Healthy.String(), activePeerCount)
-	case count < 5:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 5 peers are connected")
+	case count < 10:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 10 peers are connected")
 	case count == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "error: no peers are connected")
 	}
diff --git a/nodeprobe/nodeprobe.go b/nodeprobe/nodeprobe.go
index d24c7d858c..228df78577 100644
--- a/nodeprobe/nodeprobe.go
+++ b/nodeprobe/nodeprobe.go
@@ -138,6 +138,8 @@ func (p *Prober) AddNode(name string, node Node) {
 }
 
 func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, p.interval)
+	defer cancel()
 	err := p.nodes["consensus client"].Healthy(ctx)
 	if err != nil {
 		return err
@@ -146,6 +148,8 @@ func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
 }
 
 func (p *Prober) CheckExecutionNodeHealth(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, p.interval)
+	defer cancel()
 	err := p.nodes["execution client"].Healthy(ctx)
 	if err != nil {
 		return err
@@ -153,7 +157,9 @@ func (p *Prober) CheckExecutionNodeHealth(ctx context.Context) error {
 	return nil
 }
 
-func (p *Prober) CheckEventSycNodeHealth(ctx context.Context) error {
+func (p *Prober) CheckEventSyncerHealth(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, p.interval)
+	defer cancel()
 	err := p.nodes["event syncer"].Healthy(ctx)
 	if err != nil {
 		return err

From 2f54f4e9c312f7c67abb7a0f7e30af5f86ca5665 Mon Sep 17 00:00:00 2001
From: Matus Kysel <matus@blox.io>
Date: Tue, 14 Nov 2023 10:26:45 +0100
Subject: [PATCH 08/34] deploy to stage

---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 627b5f0e8c..46ea1f99ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,7 +40,7 @@ Build stage Docker image:
     - docker tag $IMAGE_NAME:$CI_COMMIT_SHA $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
     - $DOCKER_LOGIN_TO_INFRA_STAGE_REPO && docker push $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
   only:
-    - stage
+    - node_health
 
 # +---------------------+
 # | STAGE HETZNER NODES |
@@ -64,26 +64,26 @@ Deploy nodes to hetzner stage:
     # +--------------------+
     # |  Deploy SSV nodes  |
     # +--------------------+
-    - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
     - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
   only:
-    - stage
+    - node_health
 
 Deploy exporter to hetzner stage:
   stage: deploy

From 00566c5c1391a453d0e4c33528430e47f4f8b083 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 14 Nov 2023 17:23:51 +0600
Subject: [PATCH 09/34] add plaintext response

---
 api/handlers/node.go | 28 ++++++++++++++++++++++------
 api/handling.go      | 22 +++++++++++++++++++++-
 go.mod               |  1 +
 go.sum               | 31 +++++++++++++++++++++++++++++++
 4 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 69e466d643..55a2f2ea71 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
 
 	"github.com/bloxapp/ssv/api"
 	networkpeers "github.com/bloxapp/ssv/network/peers"
@@ -64,11 +65,11 @@ type identityJSON struct {
 }
 
 type healthCheckJSON struct {
-	PeersHealthStatus               string   `json:"peers_status"`
-	BeaconConnectionHealthStatus    string   `json:"beacon_health_status"`
-	ExecutionConnectionHealthStatus string   `json:"execution_health_status"`
-	EventSyncHealthStatus           string   `json:"event_sync_health_status"`
-	LocalPortsListening             []string `json:"local_ports_listening"`
+	PeersHealthStatus               string `json:"peers_status"`
+	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
+	ExecutionConnectionHealthStatus string `json:"execution_health_status"`
+	EventSyncHealthStatus           string `json:"event_sync_health_status"`
+	LocalPortsListening             string `json:"local_port_listening"`
 }
 
 type Node struct {
@@ -120,7 +121,10 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	// check ports being used
 	addrs := h.Network.ListenAddresses()
 	for _, addr := range addrs {
-		resp.LocalPortsListening = append(resp.LocalPortsListening, addr.String())
+		if addr.String() == "/p2p-circuit" || addr.Decapsulate(multiaddr.StringCast("/ip4/0.0.0.0")) == nil {
+			continue
+		}
+		resp.LocalPortsListening = addr.String()
 	}
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
@@ -153,6 +157,18 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	case count == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "error: no peers are connected")
 	}
+
+	// Handle plain text content.
+	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
+		str := fmt.Sprintf("%s: %s; %s: %s; %s: %s; %s: %s; %s: %s; \n",
+			"peers_status", resp.PeersHealthStatus,
+			"beacon_health_status", resp.BeaconConnectionHealthStatus,
+			"execution_health_status", resp.ExecutionConnectionHealthStatus,
+			"event_sync_health_status", resp.EventSyncHealthStatus,
+			"local_port_listening", resp.LocalPortsListening,
+		)
+		return api.Render(w, r, str)
+	}
 	return api.Render(w, r, resp)
 }
 
diff --git a/api/handling.go b/api/handling.go
index 385091fd06..75cdb62127 100644
--- a/api/handling.go
+++ b/api/handling.go
@@ -4,6 +4,12 @@ import (
 	"net/http"
 
 	"github.com/go-chi/render"
+	"github.com/golang/gddo/httputil"
+)
+
+const (
+	ContentTypePlainText = "text/plain"
+	ContentTypeJSON      = "application/json"
 )
 
 type HandlerFunc func(http.ResponseWriter, *http.Request) error
@@ -23,6 +29,20 @@ func Handler(h HandlerFunc) http.HandlerFunc {
 }
 
 func Render(w http.ResponseWriter, r *http.Request, response any) error {
-	render.JSON(w, r, response)
+	switch NegotiateContentType(r) {
+	case ContentTypePlainText:
+		render.PlainText(w, r, response.(string))
+	case ContentTypeJSON:
+		render.JSON(w, r, response)
+	}
 	return nil
 }
+
+// negotiateContentType parses "Accept:" header and returns preferred content type string.
+func NegotiateContentType(r *http.Request) string {
+	contentTypes := []string{
+		ContentTypePlainText,
+		ContentTypeJSON,
+	}
+	return httputil.NegotiateContentType(r, contentTypes, ContentTypePlainText)
+}
diff --git a/go.mod b/go.mod
index 7be627fb25..015cf2892f 100644
--- a/go.mod
+++ b/go.mod
@@ -16,6 +16,7 @@ require (
 	github.com/ferranbt/fastssz v0.1.3
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/go-chi/render v1.0.2
+	github.com/golang/gddo v0.0.0-20200528160355-8d077c1d8f4c
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/websocket v1.5.0
diff --git a/go.sum b/go.sum
index 527b7eea50..23bb093fa1 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,4 @@
+cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -59,6 +60,7 @@ github.com/bloxapp/eth2-key-manager v1.3.2/go.mod h1:cT+qAJfnAzNz9StFoHQ8xAkyU2e
 github.com/bloxapp/ssv-spec v0.3.4 h1:uu1pAP8FBucGf1FGORjzqz7if0vWGRY5w6ILLhA7IuM=
 github.com/bloxapp/ssv-spec v0.3.4/go.mod h1:zPJR7YnG5iZ6I0h6EzfVly8bTBXaZwcx4TyJ8pzYVd8=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
+github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U=
 github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
@@ -169,11 +171,13 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays=
@@ -207,6 +211,7 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/validator/v10 v10.13.0 h1:cFRQdfaSMCOSfGCCLB20MHvuoHb/s5G8L5pu2ppK5AQ=
+github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
@@ -236,11 +241,14 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog=
 github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang/gddo v0.0.0-20200528160355-8d077c1d8f4c h1:HoqgYR60VYu5+0BuG6pjeGp7LKEPZnHt+dUClx9PeIs=
+github.com/golang/gddo v0.0.0-20200528160355-8d077c1d8f4c/go.mod h1:sam69Hju0uq+5uvLJUMDlsKlQ21Vrs1Kd/1YFPNYdOU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -262,6 +270,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
@@ -270,6 +279,7 @@ github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85q
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
 github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -304,6 +314,7 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRid
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
@@ -321,6 +332,7 @@ github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuW
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
 github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/herumi/bls-eth-go-binary v0.0.0-20210130185500-57372fb27371/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U=
 github.com/herumi/bls-eth-go-binary v1.29.1 h1:XcNSHYTyNjEUVfWDCE2gtG5r95biTwd7MJUJF09LtSE=
@@ -339,6 +351,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/ilyakaznacheev/cleanenv v1.4.2 h1:nRqiriLMAC7tz7GzjzUTBHfzdzw6SQ7XvTagkFqe/zU=
 github.com/ilyakaznacheev/cleanenv v1.4.2/go.mod h1:i0owW+HDxeGKE0/JPREJOdSCPIyOnmh6C0xhWAkF/xA=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -458,10 +471,12 @@ github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5
 github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q=
 github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk=
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
+github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -469,6 +484,7 @@ github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
@@ -506,6 +522,7 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -593,6 +610,7 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -714,15 +732,20 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
 github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
@@ -888,6 +911,7 @@ golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -909,12 +933,14 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
+golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -992,6 +1018,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1032,6 +1059,7 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
 gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
+google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1040,7 +1068,9 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -1052,6 +1082,7 @@ google.golang.org/genproto v0.0.0-20200218151345-dad8c97a84f5/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
 google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
+google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=

From 5f9df91897a509457268e4d07a87a4ab333363f4 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 14 Nov 2023 17:41:15 +0600
Subject: [PATCH 10/34] lint

---
 api/handlers/node.go | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 55a2f2ea71..a030ebae1f 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -14,6 +14,8 @@ import (
 	"github.com/bloxapp/ssv/nodeprobe"
 )
 
+const healthyPeersAmount = 10
+
 type TopicIndex interface {
 	PeersByTopic() ([]peer.ID, map[string][]peer.ID)
 }
@@ -93,8 +95,8 @@ func (h *Node) Identity(w http.ResponseWriter, r *http.Request) error {
 }
 
 func (h *Node) Peers(w http.ResponseWriter, r *http.Request) error {
-	prs := h.Network.Peers()
-	resp := h.peers(prs)
+	peers := h.Network.Peers()
+	resp := h.peers(peers)
 	return api.Render(w, r, resp)
 }
 
@@ -102,8 +104,8 @@ func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
 	allpeers, peerbytpc := h.TopicIndex.PeersByTopic()
 	alland := AllPeersAndTopicsJSON{}
 	tpcs := []topicIndexJSON{}
-	for topic, peerz := range peerbytpc {
-		tpcs = append(tpcs, topicIndexJSON{TopicName: topic, Peers: peerz})
+	for topic, peers := range peerbytpc {
+		tpcs = append(tpcs, topicIndexJSON{TopicName: topic, Peers: peers})
 	}
 	alland.AllPeers = allpeers
 	alland.PeersByTopic = tpcs
@@ -129,35 +131,34 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
 	if err != nil {
-		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
+		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
 	if err != nil {
-		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
+		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
 	}
 	// check event sync health
 	err = h.NodeProber.CheckEventSyncerHealth(ctx)
 	if err != nil {
-		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), err.Error())
+		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
 	}
 	// check peers connection
 	var activePeerCount int
-	prs := h.Network.Peers()
-	for _, p := range h.peers(prs) {
+	peers := h.Network.Peers()
+	for _, p := range h.peers(peers) {
 		if p.Connectedness == "Connected" {
 			activePeerCount++
 		}
 	}
-	switch count := activePeerCount; {
-	case count >= 10:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %d  peers are connected", Healthy.String(), activePeerCount)
-	case count < 10:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "less than 10 peers are connected")
-	case count == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy.String(), "error: no peers are connected")
+	switch {
+	case activePeerCount >= healthyPeersAmount:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %d  peers are connected", Healthy, activePeerCount)
+	case activePeerCount > 0 && activePeerCount < healthyPeersAmount:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: less than %d peers are connected", NotHealthy, healthyPeersAmount)
+	case activePeerCount == 0:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, "error: no peers are connected")
 	}
-
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
 		str := fmt.Sprintf("%s: %s; %s: %s; %s: %s; %s: %s; %s: %s; \n",

From 8f1e41539bc94bf12ec8068048a7b2fbfa90e616 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 14 Nov 2023 17:49:41 +0600
Subject: [PATCH 11/34] lint

---
 nodeprobe/nodeprobe.go | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/nodeprobe/nodeprobe.go b/nodeprobe/nodeprobe.go
index 228df78577..7beeb815c0 100644
--- a/nodeprobe/nodeprobe.go
+++ b/nodeprobe/nodeprobe.go
@@ -140,29 +140,17 @@ func (p *Prober) AddNode(name string, node Node) {
 func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
-	err := p.nodes["consensus client"].Healthy(ctx)
-	if err != nil {
-		return err
-	}
-	return nil
+	return p.nodes["consensus client"].Healthy(ctx)
 }
 
 func (p *Prober) CheckExecutionNodeHealth(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
-	err := p.nodes["execution client"].Healthy(ctx)
-	if err != nil {
-		return err
-	}
-	return nil
+	return p.nodes["execution client"].Healthy(ctx)
 }
 
 func (p *Prober) CheckEventSyncerHealth(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
-	err := p.nodes["event syncer"].Healthy(ctx)
-	if err != nil {
-		return err
-	}
-	return nil
+	return p.nodes["event syncer"].Healthy(ctx)
 }

From a04b9d05a18a408646aae2efe896d630ddb0ca0e Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Wed, 15 Nov 2023 11:16:45 +0600
Subject: [PATCH 12/34] change to good/bad

---
 api/handlers/node.go | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index a030ebae1f..ba0c3bdd96 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -23,12 +23,12 @@ type TopicIndex interface {
 type HealthStatus int
 
 const (
-	NotHealthy HealthStatus = iota
-	Healthy
+	Bad HealthStatus = iota
+	Good
 )
 
 func (c HealthStatus) String() string {
-	str := [...]string{"NotHealthy", "Healthy"}
+	str := [...]string{"Bad", "Good"}
 	if c < 0 || int(c) >= len(str) {
 		return "(unrecognized)"
 	}
@@ -116,9 +116,10 @@ func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
 func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	ctx := context.Background()
 	resp := healthCheckJSON{
-		BeaconConnectionHealthStatus:    Healthy.String(),
-		ExecutionConnectionHealthStatus: Healthy.String(),
-		EventSyncHealthStatus:           Healthy.String(),
+		BeaconConnectionHealthStatus:    Good.String(),
+		ExecutionConnectionHealthStatus: Good.String(),
+		EventSyncHealthStatus:           Good.String(),
+		PeersHealthStatus:               Good.String(),
 	}
 	// check ports being used
 	addrs := h.Network.ListenAddresses()
@@ -131,17 +132,17 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
 	if err != nil {
-		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
+		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
 	if err != nil {
-		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
+		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
 	}
 	// check event sync health
 	err = h.NodeProber.CheckEventSyncerHealth(ctx)
 	if err != nil {
-		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, err.Error())
+		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
 	}
 	// check peers connection
 	var activePeerCount int
@@ -152,14 +153,12 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 	switch {
-	case activePeerCount >= healthyPeersAmount:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %d  peers are connected", Healthy, activePeerCount)
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: less than %d peers are connected", NotHealthy, healthyPeersAmount)
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %d peers are connected", Bad, activePeerCount)
 	case activePeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", NotHealthy, "error: no peers are connected")
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", Bad, "error: no peers are connected")
 	}
-	// Handle plain text content.
+	// handle plain text content
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
 		str := fmt.Sprintf("%s: %s; %s: %s; %s: %s; %s: %s; %s: %s; \n",
 			"peers_status", resp.PeersHealthStatus,

From 7dced1222db6324d3e480c54a0ed272d985900d1 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Wed, 15 Nov 2023 11:29:53 +0600
Subject: [PATCH 13/34] lint

---
 api/handlers/node.go   | 30 +++++++++++++++---------------
 nodeprobe/nodeprobe.go |  2 ++
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index ba0c3bdd96..2bf01189f1 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -20,15 +20,15 @@ type TopicIndex interface {
 	PeersByTopic() ([]peer.ID, map[string][]peer.ID)
 }
 
-type HealthStatus int
+type healthStatus int
 
 const (
-	Bad HealthStatus = iota
-	Good
+	bad healthStatus = iota
+	good
 )
 
-func (c HealthStatus) String() string {
-	str := [...]string{"Bad", "Good"}
+func (c healthStatus) String() string {
+	str := [...]string{"bad", "good"}
 	if c < 0 || int(c) >= len(str) {
 		return "(unrecognized)"
 	}
@@ -116,10 +116,10 @@ func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
 func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	ctx := context.Background()
 	resp := healthCheckJSON{
-		BeaconConnectionHealthStatus:    Good.String(),
-		ExecutionConnectionHealthStatus: Good.String(),
-		EventSyncHealthStatus:           Good.String(),
-		PeersHealthStatus:               Good.String(),
+		BeaconConnectionHealthStatus:    good.String(),
+		ExecutionConnectionHealthStatus: good.String(),
+		EventSyncHealthStatus:           good.String(),
+		PeersHealthStatus:               good.String(),
 	}
 	// check ports being used
 	addrs := h.Network.ListenAddresses()
@@ -132,17 +132,17 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	// check consensus node health
 	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
 	if err != nil {
-		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
+		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
 	}
 	// check execution node health
 	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
 	if err != nil {
-		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
+		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
 	}
 	// check event sync health
 	err = h.NodeProber.CheckEventSyncerHealth(ctx)
 	if err != nil {
-		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", Bad, err.Error())
+		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
 	}
 	// check peers connection
 	var activePeerCount int
@@ -154,13 +154,13 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	switch {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %d peers are connected", Bad, activePeerCount)
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %d peers are connected", bad, activePeerCount)
 	case activePeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", Bad, "error: no peers are connected")
+		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", bad, "error: no peers are connected")
 	}
 	// handle plain text content
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s; %s: %s; %s: %s; %s: %s; %s: %s; \n",
+		str := fmt.Sprintf("%s: %s\n %s: %s\n %s: %s\n %s: %s\n %s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,
 			"execution_health_status", resp.ExecutionConnectionHealthStatus,
diff --git a/nodeprobe/nodeprobe.go b/nodeprobe/nodeprobe.go
index 7beeb815c0..d4f0ca03b2 100644
--- a/nodeprobe/nodeprobe.go
+++ b/nodeprobe/nodeprobe.go
@@ -137,6 +137,8 @@ func (p *Prober) AddNode(name string, node Node) {
 	p.nodes[name] = node
 }
 
+// TODO: string constants are error-prone.
+// Add a method to clients that returns their name or solve this in another way
 func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()

From 98bbf47f2f128f15c7ec74710271ab95c2130e6f Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Wed, 15 Nov 2023 13:41:09 +0600
Subject: [PATCH 14/34] lint

---
 api/handlers/node.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 2bf01189f1..bb29b71e25 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -160,7 +160,7 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// handle plain text content
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n %s: %s\n %s: %s\n %s: %s\n %s: %s\n",
+		str := fmt.Sprintf("%s: %s\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,
 			"execution_health_status", resp.ExecutionConnectionHealthStatus,

From 7b06e570936944f06f2d49557fe31bac1bd200d3 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Wed, 15 Nov 2023 17:01:17 +0600
Subject: [PATCH 15/34] refactor

---
 api/handlers/node.go | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index bb29b71e25..b22b02b075 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -121,7 +121,7 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 		EventSyncHealthStatus:           good.String(),
 		PeersHealthStatus:               good.String(),
 	}
-	// check ports being used
+	// Check ports being used.
 	addrs := h.Network.ListenAddresses()
 	for _, addr := range addrs {
 		if addr.String() == "/p2p-circuit" || addr.Decapsulate(multiaddr.StringCast("/ip4/0.0.0.0")) == nil {
@@ -129,22 +129,11 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 		}
 		resp.LocalPortsListening = addr.String()
 	}
-	// check consensus node health
-	err := h.NodeProber.CheckBeaconNodeHealth(ctx)
-	if err != nil {
-		resp.BeaconConnectionHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
-	}
-	// check execution node health
-	err = h.NodeProber.CheckExecutionNodeHealth(ctx)
-	if err != nil {
-		resp.ExecutionConnectionHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
-	}
-	// check event sync health
-	err = h.NodeProber.CheckEventSyncerHealth(ctx)
-	if err != nil {
-		resp.EventSyncHealthStatus = fmt.Sprintf("%s: %s", bad, err.Error())
-	}
-	// check peers connection
+	// Performing various health checks.
+	resp.BeaconConnectionHealthStatus = performHealthCheck(h.NodeProber.CheckBeaconNodeHealth, ctx)
+	resp.ExecutionConnectionHealthStatus = performHealthCheck(h.NodeProber.CheckExecutionNodeHealth, ctx)
+	resp.EventSyncHealthStatus = performHealthCheck(h.NodeProber.CheckEventSyncerHealth, ctx)
+	// Check peers connection.
 	var activePeerCount int
 	peers := h.Network.Peers()
 	for _, p := range h.peers(peers) {
@@ -158,7 +147,7 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	case activePeerCount == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", bad, "error: no peers are connected")
 	}
-	// handle plain text content
+	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
 		str := fmt.Sprintf("%s: %s\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
@@ -201,3 +190,10 @@ func (h *Node) peers(peers []peer.ID) []peerJSON {
 	}
 	return resp
 }
+
+func performHealthCheck(healthCheckFunc func(context.Context) error, ctx context.Context) string {
+	if err := healthCheckFunc(ctx); err != nil {
+		return fmt.Sprintf("%s: %s", bad, err.Error())
+	}
+	return good.String()
+}

From 17c273f57a183275b82e2a1131f41b73cfda9751 Mon Sep 17 00:00:00 2001
From: Matus Kysel <matus@blox.io>
Date: Wed, 15 Nov 2023 17:36:30 +0100
Subject: [PATCH 16/34] Revert "deploy to stage"

This reverts commit 2f54f4e9c312f7c67abb7a0f7e30af5f86ca5665.
---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 46ea1f99ff..627b5f0e8c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,7 +40,7 @@ Build stage Docker image:
     - docker tag $IMAGE_NAME:$CI_COMMIT_SHA $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
     - $DOCKER_LOGIN_TO_INFRA_STAGE_REPO && docker push $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
   only:
-    - node_health
+    - stage
 
 # +---------------------+
 # | STAGE HETZNER NODES |
@@ -64,26 +64,26 @@ Deploy nodes to hetzner stage:
     # +--------------------+
     # |  Deploy SSV nodes  |
     # +--------------------+
-    # - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
     - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
   only:
-    - node_health
+    - stage
 
 Deploy exporter to hetzner stage:
   stage: deploy

From 4f4b0e3bab531398ccc1eb06b76090ff31776951 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Thu, 16 Nov 2023 19:22:30 +0600
Subject: [PATCH 17/34] lint

---
 api/handlers/node.go | 2 +-
 cli/operator/node.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index b22b02b075..dda15fdd11 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -145,7 +145,7 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: %d peers are connected", bad, activePeerCount)
 	case activePeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %s", bad, "error: no peers are connected")
+		resp.PeersHealthStatus = fmt.Sprintf("%s: error: no peers are connected", bad)
 	}
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
diff --git a/cli/operator/node.go b/cli/operator/node.go
index 44f6a6aea5..2634815b0d 100644
--- a/cli/operator/node.go
+++ b/cli/operator/node.go
@@ -12,13 +12,13 @@ import (
 	"os"
 	"time"
 
+	spectypes "github.com/bloxapp/ssv-spec/types"
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ilyakaznacheev/cleanenv"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
-	spectypes "github.com/bloxapp/ssv-spec/types"
 	"github.com/bloxapp/ssv/api/handlers"
 	apiserver "github.com/bloxapp/ssv/api/server"
 	"github.com/bloxapp/ssv/beacon/goclient"

From 63024fe8ecf2c77b2188be9508274ff815036ee0 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 17 Nov 2023 15:41:17 +0600
Subject: [PATCH 18/34] add inbound/outbound count for health + deploy to stage

---
 .gitlab-ci.yml                               | 38 ++++++++++----------
 .k8/hetzner-stage/ssv-node-45-deployment.yml |  4 +--
 .k8/hetzner-stage/ssv-node-46-deployment.yml |  4 +--
 .k8/hetzner-stage/ssv-node-47-deployment.yml |  8 ++---
 api/handlers/node.go                         | 35 ++++++++++++++++--
 5 files changed, 59 insertions(+), 30 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 627b5f0e8c..ecc92d533d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,7 +40,7 @@ Build stage Docker image:
     - docker tag $IMAGE_NAME:$CI_COMMIT_SHA $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
     - $DOCKER_LOGIN_TO_INFRA_STAGE_REPO && docker push $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
   only:
-    - stage
+    - node_health
 
 # +---------------------+
 # | STAGE HETZNER NODES |
@@ -64,26 +64,26 @@ Deploy nodes to hetzner stage:
     # +--------------------+
     # |  Deploy SSV nodes  |
     # +--------------------+
-    - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
     - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    # - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
   only:
-    - stage
+    - node_health
 
 Deploy exporter to hetzner stage:
   stage: deploy
diff --git a/.k8/hetzner-stage/ssv-node-45-deployment.yml b/.k8/hetzner-stage/ssv-node-45-deployment.yml
index dd4e94430d..ada242a32b 100644
--- a/.k8/hetzner-stage/ssv-node-45-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-45-deployment.yml
@@ -14,9 +14,9 @@ spec:
       protocol: UDP
       targetPort: 12045
       name: port-12045
-    - port: 13045
+    - port: 43045
       protocol: TCP
-      targetPort: 13045
+      targetPort: 43045
       name: port-13045
     - port: 15045
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-46-deployment.yml b/.k8/hetzner-stage/ssv-node-46-deployment.yml
index 8ce5fc8625..20074ee2e0 100644
--- a/.k8/hetzner-stage/ssv-node-46-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-46-deployment.yml
@@ -10,9 +10,9 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 12046
+    - port: 42046
       protocol: UDP
-      targetPort: 12046
+      targetPort: 42046
       name: port-12046
     - port: 13046
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-47-deployment.yml b/.k8/hetzner-stage/ssv-node-47-deployment.yml
index 20f13789b7..684af0ff84 100644
--- a/.k8/hetzner-stage/ssv-node-47-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-47-deployment.yml
@@ -10,13 +10,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 12047
+    - port: 42047
       protocol: UDP
-      targetPort: 12047
+      targetPort: 42047
       name: port-12047
-    - port: 13047
+    - port: 43047
       protocol: TCP
-      targetPort: 13047
+      targetPort: 43047
       name: port-13047
     - port: 15047
       protocol: TCP
diff --git a/api/handlers/node.go b/api/handlers/node.go
index dda15fdd11..4adf8ada0f 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -68,6 +68,10 @@ type identityJSON struct {
 
 type healthCheckJSON struct {
 	PeersHealthStatus               string `json:"peers_status"`
+	PeersCount                      int    `json:"peers_count"`
+	ActivePeersCount                int    `json:"active_peers"`
+	InboundPeersCount               int    `json:"inbound_peers"`
+	OutboundPeersCount              int    `json:"outbound_peers"`
 	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
 	ExecutionConnectionHealthStatus string `json:"execution_health_status"`
 	EventSyncHealthStatus           string `json:"event_sync_health_status"`
@@ -135,22 +139,47 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	resp.EventSyncHealthStatus = performHealthCheck(h.NodeProber.CheckEventSyncerHealth, ctx)
 	// Check peers connection.
 	var activePeerCount int
+	var inboundPeerCount int
+	var outboundPeerCount int
 	peers := h.Network.Peers()
+	conns := h.Network.ConnsToPeer(h.Network.LocalPeer())
 	for _, p := range h.peers(peers) {
 		if p.Connectedness == "Connected" {
 			activePeerCount++
 		}
+		// Check connection direction.
+		for _, conn := range conns {
+			if conn.ID() == string(p.ID) {
+				switch conn.Stat().Direction {
+				case network.DirInbound:
+					inboundPeerCount++
+				case network.DirOutbound:
+					outboundPeerCount++
+				}
+			}
+		}
 	}
+
+	resp.InboundPeersCount = inboundPeerCount
+	resp.OutboundPeersCount = outboundPeerCount
+	resp.PeersCount = len(peers)
+	resp.ActivePeersCount = activePeerCount
 	switch {
-	case activePeerCount > 0 && activePeerCount < healthyPeersAmount:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: %d peers are connected", bad, activePeerCount)
+	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: not enough connected peers", bad)
+	case activePeerCount > 0 && inboundPeerCount == 0:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: error: local port is not reachable, please check the configuration", bad)
 	case activePeerCount == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: error: no peers are connected", bad)
 	}
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
+		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
+			"peers_count", resp.PeersCount,
+			"active_peers", resp.ActivePeersCount,
+			"inbound_peers", resp.InboundPeersCount,
+			"outbound_peers", resp.OutboundPeersCount,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,
 			"execution_health_status", resp.ExecutionConnectionHealthStatus,
 			"event_sync_health_status", resp.EventSyncHealthStatus,

From 596fb31450684c45eba4168b0bdffcd7ea1a7992 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 17 Nov 2023 16:54:15 +0600
Subject: [PATCH 19/34] change ports back

---
 .k8/hetzner-stage/ssv-node-45-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-46-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-47-deployment.yml | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.k8/hetzner-stage/ssv-node-45-deployment.yml b/.k8/hetzner-stage/ssv-node-45-deployment.yml
index ada242a32b..dd4e94430d 100644
--- a/.k8/hetzner-stage/ssv-node-45-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-45-deployment.yml
@@ -14,9 +14,9 @@ spec:
       protocol: UDP
       targetPort: 12045
       name: port-12045
-    - port: 43045
+    - port: 13045
       protocol: TCP
-      targetPort: 43045
+      targetPort: 13045
       name: port-13045
     - port: 15045
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-46-deployment.yml b/.k8/hetzner-stage/ssv-node-46-deployment.yml
index 20074ee2e0..8ce5fc8625 100644
--- a/.k8/hetzner-stage/ssv-node-46-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-46-deployment.yml
@@ -10,9 +10,9 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 42046
+    - port: 12046
       protocol: UDP
-      targetPort: 42046
+      targetPort: 12046
       name: port-12046
     - port: 13046
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-47-deployment.yml b/.k8/hetzner-stage/ssv-node-47-deployment.yml
index 684af0ff84..20f13789b7 100644
--- a/.k8/hetzner-stage/ssv-node-47-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-47-deployment.yml
@@ -10,13 +10,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 42047
+    - port: 12047
       protocol: UDP
-      targetPort: 42047
+      targetPort: 12047
       name: port-12047
-    - port: 43047
+    - port: 13047
       protocol: TCP
-      targetPort: 43047
+      targetPort: 13047
       name: port-13047
     - port: 15047
       protocol: TCP

From 9203c3008f7f363777223e11e017a0a6801b5772 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 17 Nov 2023 18:16:22 +0600
Subject: [PATCH 20/34] update count

---
 api/handlers/node.go | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 4adf8ada0f..d016d5f3ba 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -3,11 +3,13 @@ package handlers
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/multiformats/go-multiaddr"
+	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
 
 	"github.com/bloxapp/ssv/api"
 	networkpeers "github.com/bloxapp/ssv/network/peers"
@@ -78,6 +80,10 @@ type healthCheckJSON struct {
 	LocalPortsListening             string `json:"local_port_listening"`
 }
 
+type pingPort struct {
+	URL string `json:"url_string"`
+}
+
 type Node struct {
 	PeersIndex networkpeers.Index
 	TopicIndex TopicIndex
@@ -127,11 +133,16 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// Check ports being used.
 	addrs := h.Network.ListenAddresses()
+	var localAddressPort net.Addr
 	for _, addr := range addrs {
-		if addr.String() == "/p2p-circuit" || addr.Decapsulate(multiaddr.StringCast("/ip4/0.0.0.0")) == nil {
+		var err error
+		if addr.String() == "/p2p-circuit" || addr.Decapsulate(ma.StringCast("/ip4/0.0.0.0")) == nil {
 			continue
 		}
-		resp.LocalPortsListening = addr.String()
+		localAddressPort, err = manet.ToNetAddr(addr)
+		if err != nil {
+			return err
+		}
 	}
 	// Performing various health checks.
 	resp.BeaconConnectionHealthStatus = performHealthCheck(h.NodeProber.CheckBeaconNodeHealth, ctx)
@@ -147,27 +158,25 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 		if p.Connectedness == "Connected" {
 			activePeerCount++
 		}
-		// Check connection direction.
-		for _, conn := range conns {
-			if conn.ID() == string(p.ID) {
-				switch conn.Stat().Direction {
-				case network.DirInbound:
-					inboundPeerCount++
-				case network.DirOutbound:
-					outboundPeerCount++
-				}
-			}
+	}
+	// Check connection direction.
+	for _, conn := range conns {
+		switch conn.Stat().Direction {
+		case network.DirInbound:
+			inboundPeerCount++
+		case network.DirOutbound:
+			outboundPeerCount++
 		}
 	}
-
 	resp.InboundPeersCount = inboundPeerCount
 	resp.OutboundPeersCount = outboundPeerCount
 	resp.PeersCount = len(peers)
 	resp.ActivePeersCount = activePeerCount
+	resp.LocalPortsListening = localAddressPort.String()
 	switch {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: not enough connected peers", bad)
-	case activePeerCount > 0 && inboundPeerCount == 0:
+	case inboundPeerCount == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: error: local port is not reachable, please check the configuration", bad)
 	case activePeerCount == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: error: no peers are connected", bad)

From aab1a1f48ff26a71a0d80cb1531ce6759731b569 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 17 Nov 2023 18:23:45 +0600
Subject: [PATCH 21/34] lint

---
 api/handlers/node.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index d016d5f3ba..82c54c4910 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -80,10 +80,6 @@ type healthCheckJSON struct {
 	LocalPortsListening             string `json:"local_port_listening"`
 }
 
-type pingPort struct {
-	URL string `json:"url_string"`
-}
-
 type Node struct {
 	PeersIndex networkpeers.Index
 	TopicIndex TopicIndex

From e3ef2d91204de1a4905ffa95474aa2dad6ec609e Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Fri, 17 Nov 2023 19:17:45 +0600
Subject: [PATCH 22/34] update conns

---
 api/handlers/node.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 82c54c4910..80b85df544 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -149,14 +149,13 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	var inboundPeerCount int
 	var outboundPeerCount int
 	peers := h.Network.Peers()
-	conns := h.Network.ConnsToPeer(h.Network.LocalPeer())
 	for _, p := range h.peers(peers) {
 		if p.Connectedness == "Connected" {
 			activePeerCount++
 		}
 	}
 	// Check connection direction.
-	for _, conn := range conns {
+	for _, conn := range h.Network.Conns() {
 		switch conn.Stat().Direction {
 		case network.DirInbound:
 			inboundPeerCount++

From 553fd0ac1b2c7359470f2888f2baa431ac7e54a2 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 20 Nov 2023 14:40:06 +0600
Subject: [PATCH 23/34] lint

---
 api/handlers/node.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 80b85df544..6c61431472 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -171,8 +171,8 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	switch {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: not enough connected peers", bad)
-	case inboundPeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: error: local port is not reachable, please check the configuration", bad)
+	case activePeerCount > 0 && inboundPeerCount == 0:
+		resp.PeersHealthStatus = fmt.Sprintf("%s: error: local port probably is not reachable, please check the configuration", bad)
 	case activePeerCount == 0:
 		resp.PeersHealthStatus = fmt.Sprintf("%s: error: no peers are connected", bad)
 	}

From 79e2b9448f06d15c33ec93af894279fbe80af161 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 15:36:29 +0600
Subject: [PATCH 24/34] remove connected peer count

---
 api/handlers/node.go | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 6c61431472..671c58f503 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -71,7 +71,6 @@ type identityJSON struct {
 type healthCheckJSON struct {
 	PeersHealthStatus               string `json:"peers_status"`
 	PeersCount                      int    `json:"peers_count"`
-	ActivePeersCount                int    `json:"active_peers"`
 	InboundPeersCount               int    `json:"inbound_peers"`
 	OutboundPeersCount              int    `json:"outbound_peers"`
 	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
@@ -148,12 +147,6 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	var activePeerCount int
 	var inboundPeerCount int
 	var outboundPeerCount int
-	peers := h.Network.Peers()
-	for _, p := range h.peers(peers) {
-		if p.Connectedness == "Connected" {
-			activePeerCount++
-		}
-	}
 	// Check connection direction.
 	for _, conn := range h.Network.Conns() {
 		switch conn.Stat().Direction {
@@ -165,8 +158,7 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	resp.InboundPeersCount = inboundPeerCount
 	resp.OutboundPeersCount = outboundPeerCount
-	resp.PeersCount = len(peers)
-	resp.ActivePeersCount = activePeerCount
+	resp.PeersCount = len(h.Network.Peers())
 	resp.LocalPortsListening = localAddressPort.String()
 	switch {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
@@ -178,10 +170,9 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
+		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
 			"peers_count", resp.PeersCount,
-			"active_peers", resp.ActivePeersCount,
 			"inbound_peers", resp.InboundPeersCount,
 			"outbound_peers", resp.OutboundPeersCount,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,

From 6fc92822cdce851138446d8c190c01217a872136 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 15:36:51 +0600
Subject: [PATCH 25/34] test blocked ports

---
 .k8/hetzner-stage/ssv-node-46-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-47-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-48-deployment.yml | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.k8/hetzner-stage/ssv-node-46-deployment.yml b/.k8/hetzner-stage/ssv-node-46-deployment.yml
index 8ce5fc8625..bedf207a16 100644
--- a/.k8/hetzner-stage/ssv-node-46-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-46-deployment.yml
@@ -14,9 +14,9 @@ spec:
       protocol: UDP
       targetPort: 12046
       name: port-12046
-    - port: 13046
+    - port: 43046
       protocol: TCP
-      targetPort: 13046
+      targetPort: 43046
       name: port-13046
     - port: 15046
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-47-deployment.yml b/.k8/hetzner-stage/ssv-node-47-deployment.yml
index 20f13789b7..b77d8493c9 100644
--- a/.k8/hetzner-stage/ssv-node-47-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-47-deployment.yml
@@ -10,9 +10,9 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 12047
+    - port: 42047
       protocol: UDP
-      targetPort: 12047
+      targetPort: 42047
       name: port-12047
     - port: 13047
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-48-deployment.yml b/.k8/hetzner-stage/ssv-node-48-deployment.yml
index e750831e12..471605ad41 100644
--- a/.k8/hetzner-stage/ssv-node-48-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-48-deployment.yml
@@ -10,13 +10,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 12048
+    - port: 42048
       protocol: UDP
-      targetPort: 12048
+      targetPort: 42048
       name: port-12048
-    - port: 13048
+    - port: 43048
       protocol: TCP
-      targetPort: 13048
+      targetPort: 43048
       name: port-13048
     - port: 15048
       protocol: TCP

From 58fae8fac91ca7065847595a7cceb77306a3c732 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 16:03:00 +0600
Subject: [PATCH 26/34] Revert "remove connected peer count"

This reverts commit 79e2b9448f06d15c33ec93af894279fbe80af161.
---
 api/handlers/node.go | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 671c58f503..6c61431472 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -71,6 +71,7 @@ type identityJSON struct {
 type healthCheckJSON struct {
 	PeersHealthStatus               string `json:"peers_status"`
 	PeersCount                      int    `json:"peers_count"`
+	ActivePeersCount                int    `json:"active_peers"`
 	InboundPeersCount               int    `json:"inbound_peers"`
 	OutboundPeersCount              int    `json:"outbound_peers"`
 	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
@@ -147,6 +148,12 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	var activePeerCount int
 	var inboundPeerCount int
 	var outboundPeerCount int
+	peers := h.Network.Peers()
+	for _, p := range h.peers(peers) {
+		if p.Connectedness == "Connected" {
+			activePeerCount++
+		}
+	}
 	// Check connection direction.
 	for _, conn := range h.Network.Conns() {
 		switch conn.Stat().Direction {
@@ -158,7 +165,8 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	resp.InboundPeersCount = inboundPeerCount
 	resp.OutboundPeersCount = outboundPeerCount
-	resp.PeersCount = len(h.Network.Peers())
+	resp.PeersCount = len(peers)
+	resp.ActivePeersCount = activePeerCount
 	resp.LocalPortsListening = localAddressPort.String()
 	switch {
 	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
@@ -170,9 +178,10 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
+		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
 			"peers_count", resp.PeersCount,
+			"active_peers", resp.ActivePeersCount,
 			"inbound_peers", resp.InboundPeersCount,
 			"outbound_peers", resp.OutboundPeersCount,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,

From a9594197e672d1ccb4772662801a90f4a40f6e12 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 17:00:57 +0600
Subject: [PATCH 27/34] leave only active peers count

---
 api/handlers/node.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 6c61431472..390c998de0 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -70,8 +70,7 @@ type identityJSON struct {
 
 type healthCheckJSON struct {
 	PeersHealthStatus               string `json:"peers_status"`
-	PeersCount                      int    `json:"peers_count"`
-	ActivePeersCount                int    `json:"active_peers"`
+	ActivePeersCount                int    `json:"peers_count"`
 	InboundPeersCount               int    `json:"inbound_peers"`
 	OutboundPeersCount              int    `json:"outbound_peers"`
 	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
@@ -165,7 +164,6 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	resp.InboundPeersCount = inboundPeerCount
 	resp.OutboundPeersCount = outboundPeerCount
-	resp.PeersCount = len(peers)
 	resp.ActivePeersCount = activePeerCount
 	resp.LocalPortsListening = localAddressPort.String()
 	switch {
@@ -178,10 +176,9 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	}
 	// Handle plain text content.
 	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
+		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
 			"peers_status", resp.PeersHealthStatus,
-			"peers_count", resp.PeersCount,
-			"active_peers", resp.ActivePeersCount,
+			"peers_count", resp.ActivePeersCount,
 			"inbound_peers", resp.InboundPeersCount,
 			"outbound_peers", resp.OutboundPeersCount,
 			"beacon_health_status", resp.BeaconConnectionHealthStatus,

From 5e47486750e6dd45b0ed65342b00c2f1cba2d32a Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 17:13:02 +0600
Subject: [PATCH 28/34] Revert "test blocked ports"

This reverts commit 6fc92822cdce851138446d8c190c01217a872136.
---
 .k8/hetzner-stage/ssv-node-46-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-47-deployment.yml | 4 ++--
 .k8/hetzner-stage/ssv-node-48-deployment.yml | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.k8/hetzner-stage/ssv-node-46-deployment.yml b/.k8/hetzner-stage/ssv-node-46-deployment.yml
index 903d21d2c4..9abe08db3b 100644
--- a/.k8/hetzner-stage/ssv-node-46-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-46-deployment.yml
@@ -14,9 +14,9 @@ spec:
       protocol: UDP
       targetPort: 12046
       name: port-12046
-    - port: 43046
+    - port: 13046
       protocol: TCP
-      targetPort: 43046
+      targetPort: 13046
       name: port-13046
     - port: 15046
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-47-deployment.yml b/.k8/hetzner-stage/ssv-node-47-deployment.yml
index 8635c5da9d..89964d30fb 100644
--- a/.k8/hetzner-stage/ssv-node-47-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-47-deployment.yml
@@ -10,9 +10,9 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 42047
+    - port: 12047
       protocol: UDP
-      targetPort: 42047
+      targetPort: 12047
       name: port-12047
     - port: 13047
       protocol: TCP
diff --git a/.k8/hetzner-stage/ssv-node-48-deployment.yml b/.k8/hetzner-stage/ssv-node-48-deployment.yml
index 77ae36c8e0..843835dd40 100644
--- a/.k8/hetzner-stage/ssv-node-48-deployment.yml
+++ b/.k8/hetzner-stage/ssv-node-48-deployment.yml
@@ -10,13 +10,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 42048
+    - port: 12048
       protocol: UDP
-      targetPort: 42048
+      targetPort: 12048
       name: port-12048
-    - port: 43048
+    - port: 13048
       protocol: TCP
-      targetPort: 43048
+      targetPort: 13048
       name: port-13048
     - port: 15048
       protocol: TCP

From dce5bb9b937f9cd722cd3d8efd8e3c5481a0a4fa Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Tue, 21 Nov 2023 18:28:31 +0600
Subject: [PATCH 29/34] ci to stage

---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ecc92d533d..627b5f0e8c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -40,7 +40,7 @@ Build stage Docker image:
     - docker tag $IMAGE_NAME:$CI_COMMIT_SHA $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
     - $DOCKER_LOGIN_TO_INFRA_STAGE_REPO && docker push $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
   only:
-    - node_health
+    - stage
 
 # +---------------------+
 # | STAGE HETZNER NODES |
@@ -64,26 +64,26 @@ Deploy nodes to hetzner stage:
     # +--------------------+
     # |  Deploy SSV nodes  |
     # +--------------------+
-    # - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
     - .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
-    # - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
+    - .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
   only:
-    - node_health
+    - stage
 
 Deploy exporter to hetzner stage:
   stage: deploy

From b86b31fbe3dabc5a49541d988d6154be40d43b32 Mon Sep 17 00:00:00 2001
From: pavelkrolevets <pavelkrolevets@gmail.com>
Date: Mon, 27 Nov 2023 15:39:22 +0600
Subject: [PATCH 30/34] add mutex to nodes access

---
 nodeprobe/nodeprobe.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/nodeprobe/nodeprobe.go b/nodeprobe/nodeprobe.go
index d4f0ca03b2..824b425a3c 100644
--- a/nodeprobe/nodeprobe.go
+++ b/nodeprobe/nodeprobe.go
@@ -140,18 +140,24 @@ func (p *Prober) AddNode(name string, node Node) {
 // TODO: string constants are error-prone.
 // Add a method to clients that returns their name or solve this in another way
 func (p *Prober) CheckBeaconNodeHealth(ctx context.Context) error {
+	p.nodesMu.Lock()
+	defer p.nodesMu.Unlock()
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
 	return p.nodes["consensus client"].Healthy(ctx)
 }
 
 func (p *Prober) CheckExecutionNodeHealth(ctx context.Context) error {
+	p.nodesMu.Lock()
+	defer p.nodesMu.Unlock()
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
 	return p.nodes["execution client"].Healthy(ctx)
 }
 
 func (p *Prober) CheckEventSyncerHealth(ctx context.Context) error {
+	p.nodesMu.Lock()
+	defer p.nodesMu.Unlock()
 	ctx, cancel := context.WithTimeout(ctx, p.interval)
 	defer cancel()
 	return p.nodes["event syncer"].Healthy(ctx)

From f2a9e1e4bb5ae23499fa08acd8c420415b8f94c2 Mon Sep 17 00:00:00 2001
From: moshe-blox <89339422+moshe-blox@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:46:29 +0200
Subject: [PATCH 31/34] refactor: node health API (#1222)

* refactor: node health API
---
 api/handlers/node.go | 166 ++++++++++++++++++-------------------------
 api/handling.go      |  38 +++++-----
 api/server/server.go |   2 +-
 3 files changed, 94 insertions(+), 112 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 390c998de0..bcb60fbd15 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -2,8 +2,9 @@ package handlers
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
-	"net"
 	"net/http"
 
 	"github.com/libp2p/go-libp2p/core/network"
@@ -16,27 +17,12 @@ import (
 	"github.com/bloxapp/ssv/nodeprobe"
 )
 
-const healthyPeersAmount = 10
+const healthyPeerCount = 30
 
 type TopicIndex interface {
 	PeersByTopic() ([]peer.ID, map[string][]peer.ID)
 }
 
-type healthStatus int
-
-const (
-	bad healthStatus = iota
-	good
-)
-
-func (c healthStatus) String() string {
-	str := [...]string{"bad", "good"}
-	if c < 0 || int(c) >= len(str) {
-		return "(unrecognized)"
-	}
-	return str[c]
-}
-
 type AllPeersAndTopicsJSON struct {
 	AllPeers     []peer.ID        `json:"all_peers"`
 	PeersByTopic []topicIndexJSON `json:"peers_by_topic"`
@@ -68,15 +54,36 @@ type identityJSON struct {
 	Version   string   `json:"version"`
 }
 
+type healthStatus struct {
+	err error
+}
+
+func (h healthStatus) MarshalJSON() ([]byte, error) {
+	if h.err == nil {
+		return json.Marshal("good")
+	}
+	return json.Marshal(fmt.Sprintf("bad: %s", h.err.Error()))
+}
+
 type healthCheckJSON struct {
-	PeersHealthStatus               string `json:"peers_status"`
-	ActivePeersCount                int    `json:"peers_count"`
-	InboundPeersCount               int    `json:"inbound_peers"`
-	OutboundPeersCount              int    `json:"outbound_peers"`
-	BeaconConnectionHealthStatus    string `json:"beacon_health_status"`
-	ExecutionConnectionHealthStatus string `json:"execution_health_status"`
-	EventSyncHealthStatus           string `json:"event_sync_health_status"`
-	LocalPortsListening             string `json:"local_port_listening"`
+	P2P           healthStatus `json:"p2p"`
+	BeaconNode    healthStatus `json:"beacon_node"`
+	ExecutionNode healthStatus `json:"execution_node"`
+	EventSyncer   healthStatus `json:"event_syncer"`
+	Advanced      struct {
+		Peers           int      `json:"peers"`
+		InboundConns    int      `json:"inbound_conns"`
+		OutboundConns   int      `json:"outbound_conns"`
+		ListenAddresses []string `json:"p2p_listen_addresses"`
+	} `json:"advanced"`
+}
+
+func (hc healthCheckJSON) String() string {
+	b, err := json.MarshalIndent(hc, "", "  ")
+	if err != nil {
+		return fmt.Sprintf("error marshalling healthCheckJSON: %s", err.Error())
+	}
+	return string(b)
 }
 
 type Node struct {
@@ -106,88 +113,64 @@ func (h *Node) Peers(w http.ResponseWriter, r *http.Request) error {
 }
 
 func (h *Node) Topics(w http.ResponseWriter, r *http.Request) error {
-	allpeers, peerbytpc := h.TopicIndex.PeersByTopic()
-	alland := AllPeersAndTopicsJSON{}
-	tpcs := []topicIndexJSON{}
-	for topic, peers := range peerbytpc {
-		tpcs = append(tpcs, topicIndexJSON{TopicName: topic, Peers: peers})
+	peers, byTopic := h.TopicIndex.PeersByTopic()
+
+	resp := AllPeersAndTopicsJSON{
+		AllPeers: peers,
+	}
+	for topic, peers := range byTopic {
+		resp.PeersByTopic = append(resp.PeersByTopic, topicIndexJSON{TopicName: topic, Peers: peers})
 	}
-	alland.AllPeers = allpeers
-	alland.PeersByTopic = tpcs
 
-	return api.Render(w, r, alland)
+	return api.Render(w, r, resp)
 }
 
 func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	ctx := context.Background()
-	resp := healthCheckJSON{
-		BeaconConnectionHealthStatus:    good.String(),
-		ExecutionConnectionHealthStatus: good.String(),
-		EventSyncHealthStatus:           good.String(),
-		PeersHealthStatus:               good.String(),
-	}
-	// Check ports being used.
-	addrs := h.Network.ListenAddresses()
-	var localAddressPort net.Addr
-	for _, addr := range addrs {
-		var err error
+	var resp healthCheckJSON
+
+	// Retrieve P2P listen addresses.
+	for _, addr := range h.Network.ListenAddresses() {
 		if addr.String() == "/p2p-circuit" || addr.Decapsulate(ma.StringCast("/ip4/0.0.0.0")) == nil {
+			// Skip circuit and non-IP4 addresses.
 			continue
 		}
-		localAddressPort, err = manet.ToNetAddr(addr)
+		netAddr, err := manet.ToNetAddr(addr)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to convert multiaddr to net.Addr: %w", err)
 		}
+		resp.Advanced.ListenAddresses = append(resp.Advanced.ListenAddresses, netAddr.String())
 	}
-	// Performing various health checks.
-	resp.BeaconConnectionHealthStatus = performHealthCheck(h.NodeProber.CheckBeaconNodeHealth, ctx)
-	resp.ExecutionConnectionHealthStatus = performHealthCheck(h.NodeProber.CheckExecutionNodeHealth, ctx)
-	resp.EventSyncHealthStatus = performHealthCheck(h.NodeProber.CheckEventSyncerHealth, ctx)
-	// Check peers connection.
-	var activePeerCount int
-	var inboundPeerCount int
-	var outboundPeerCount int
+
+	// Count peers and connections.
 	peers := h.Network.Peers()
 	for _, p := range h.peers(peers) {
 		if p.Connectedness == "Connected" {
-			activePeerCount++
+			resp.Advanced.Peers++
 		}
-	}
-	// Check connection direction.
-	for _, conn := range h.Network.Conns() {
-		switch conn.Stat().Direction {
-		case network.DirInbound:
-			inboundPeerCount++
-		case network.DirOutbound:
-			outboundPeerCount++
+		for _, conn := range p.Connections {
+			if conn.Direction == "inbound" {
+				resp.Advanced.InboundConns++
+			} else {
+				resp.Advanced.OutboundConns++
+			}
 		}
 	}
-	resp.InboundPeersCount = inboundPeerCount
-	resp.OutboundPeersCount = outboundPeerCount
-	resp.ActivePeersCount = activePeerCount
-	resp.LocalPortsListening = localAddressPort.String()
-	switch {
-	case activePeerCount > 0 && activePeerCount < healthyPeersAmount && inboundPeerCount > 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: not enough connected peers", bad)
-	case activePeerCount > 0 && inboundPeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: error: local port probably is not reachable, please check the configuration", bad)
-	case activePeerCount == 0:
-		resp.PeersHealthStatus = fmt.Sprintf("%s: error: no peers are connected", bad)
-	}
-	// Handle plain text content.
-	if contentType := api.NegotiateContentType(r); contentType == api.ContentTypePlainText {
-		str := fmt.Sprintf("%s: %s\n%s: %d\n%s: %d\n%s: %d\n%s: %s\n%s: %s\n%s: %s\n%s: %s\n",
-			"peers_status", resp.PeersHealthStatus,
-			"peers_count", resp.ActivePeersCount,
-			"inbound_peers", resp.InboundPeersCount,
-			"outbound_peers", resp.OutboundPeersCount,
-			"beacon_health_status", resp.BeaconConnectionHealthStatus,
-			"execution_health_status", resp.ExecutionConnectionHealthStatus,
-			"event_sync_health_status", resp.EventSyncHealthStatus,
-			"local_port_listening", resp.LocalPortsListening,
-		)
-		return api.Render(w, r, str)
+
+	// Report whether P2P is healthy.
+	if resp.Advanced.Peers == 0 {
+		resp.P2P = healthStatus{errors.New("no peers are connected")}
+	} else if resp.Advanced.Peers < healthyPeerCount {
+		resp.P2P = healthStatus{errors.New("not enough connected peers")}
+	} else if resp.Advanced.InboundConns == 0 {
+		resp.P2P = healthStatus{errors.New("not enough inbound connections, port is likely not reachable")}
 	}
+
+	// Check the health of Ethereum nodes and EventSyncer.
+	resp.BeaconNode = healthStatus{h.NodeProber.CheckBeaconNodeHealth(ctx)}
+	resp.ExecutionNode = healthStatus{h.NodeProber.CheckExecutionNodeHealth(ctx)}
+	resp.EventSyncer = healthStatus{(h.NodeProber.CheckEventSyncerHealth(ctx))}
+
 	return api.Render(w, r, resp)
 }
 
@@ -220,10 +203,3 @@ func (h *Node) peers(peers []peer.ID) []peerJSON {
 	}
 	return resp
 }
-
-func performHealthCheck(healthCheckFunc func(context.Context) error, ctx context.Context) string {
-	if err := healthCheckFunc(ctx); err != nil {
-		return fmt.Sprintf("%s: %s", bad, err.Error())
-	}
-	return good.String()
-}
diff --git a/api/handling.go b/api/handling.go
index 75cdb62127..218527f723 100644
--- a/api/handling.go
+++ b/api/handling.go
@@ -1,6 +1,7 @@
 package api
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/go-chi/render"
@@ -8,8 +9,8 @@ import (
 )
 
 const (
-	ContentTypePlainText = "text/plain"
-	ContentTypeJSON      = "application/json"
+	contentTypePlainText = "text/plain"
+	contentTypeJSON      = "application/json"
 )
 
 type HandlerFunc func(http.ResponseWriter, *http.Request) error
@@ -28,21 +29,26 @@ func Handler(h HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// Render negotiates the content type and renders the response, defaulting to JSON.
+// Response must implement fmt.Stringer to be rendered as plain text.
 func Render(w http.ResponseWriter, r *http.Request, response any) error {
-	switch NegotiateContentType(r) {
-	case ContentTypePlainText:
-		render.PlainText(w, r, response.(string))
-	case ContentTypeJSON:
-		render.JSON(w, r, response)
-	}
-	return nil
-}
+	// Negotiate content type, defaulting to JSON.
+	contentType := httputil.NegotiateContentType(
+		r,
+		[]string{contentTypePlainText, contentTypeJSON},
+		contentTypeJSON,
+	)
 
-// negotiateContentType parses "Accept:" header and returns preferred content type string.
-func NegotiateContentType(r *http.Request) string {
-	contentTypes := []string{
-		ContentTypePlainText,
-		ContentTypeJSON,
+	switch contentType {
+	case contentTypePlainText:
+		// Try rendering as a string, otherwise fallback to JSON.
+		if stringer, ok := response.(fmt.Stringer); ok {
+			render.PlainText(w, r, stringer.String())
+			return nil
+		}
+		fallthrough
+	default:
+		render.JSON(w, r, response)
+		return nil
 	}
-	return httputil.NegotiateContentType(r, contentTypes, ContentTypePlainText)
 }
diff --git a/api/server/server.go b/api/server/server.go
index 63e9b2c4bd..f5e2f90239 100644
--- a/api/server/server.go
+++ b/api/server/server.go
@@ -45,8 +45,8 @@ func (s *Server) Run() error {
 	router.Get("/v1/node/identity", api.Handler(s.node.Identity))
 	router.Get("/v1/node/peers", api.Handler(s.node.Peers))
 	router.Get("/v1/node/topics", api.Handler(s.node.Topics))
-	router.Get("/v1/validators", api.Handler(s.validators.List))
 	router.Get("/v1/node/health", api.Handler(s.node.Health))
+	router.Get("/v1/validators", api.Handler(s.validators.List))
 
 	s.logger.Info("Serving SSV API", zap.String("addr", s.addr))
 

From 959b4edea8f2b463ff4667ebdbdc0fdea15db879 Mon Sep 17 00:00:00 2001
From: moshe-blox <moshe@blox.io>
Date: Wed, 13 Dec 2023 16:08:44 +0200
Subject: [PATCH 32/34] added cpu_cores to healthcheck output

---
 api/handlers/node.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index bcb60fbd15..09067a7380 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"runtime"
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -75,6 +76,7 @@ type healthCheckJSON struct {
 		InboundConns    int      `json:"inbound_conns"`
 		OutboundConns   int      `json:"outbound_conns"`
 		ListenAddresses []string `json:"p2p_listen_addresses"`
+		CPUCores        int      `json:"cpu_cores"`
 	} `json:"advanced"`
 }
 
@@ -171,6 +173,9 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	resp.ExecutionNode = healthStatus{h.NodeProber.CheckExecutionNodeHealth(ctx)}
 	resp.EventSyncer = healthStatus{(h.NodeProber.CheckEventSyncerHealth(ctx))}
 
+	// Report number of CPU cores.
+	resp.Advanced.CPUCores = runtime.NumCPU()
+
 	return api.Render(w, r, resp)
 }
 

From 58379c9b9d862290f7bf185cdddcd8d1cdbef26a Mon Sep 17 00:00:00 2001
From: moshe-blox <moshe@blox.io>
Date: Wed, 20 Dec 2023 13:59:49 +0200
Subject: [PATCH 33/34] fix inbound/outbound stats

---
 api/handlers/node.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index 09067a7380..ee30783196 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -147,11 +147,11 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	// Count peers and connections.
 	peers := h.Network.Peers()
 	for _, p := range h.peers(peers) {
-		if p.Connectedness == "Connected" {
+		if p.Connectedness == network.Connected.String() {
 			resp.Advanced.Peers++
 		}
 		for _, conn := range p.Connections {
-			if conn.Direction == "inbound" {
+			if conn.Direction == network.DirInbound.String() {
 				resp.Advanced.InboundConns++
 			} else {
 				resp.Advanced.OutboundConns++

From 7d73ceca7d154e45aefe997f87584a1bc08eac62 Mon Sep 17 00:00:00 2001
From: moshe-blox <moshe@blox.io>
Date: Wed, 20 Dec 2023 14:10:55 +0200
Subject: [PATCH 34/34] Remove CPU core reporting

---
 api/handlers/node.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/api/handlers/node.go b/api/handlers/node.go
index ee30783196..a43f1124e7 100644
--- a/api/handlers/node.go
+++ b/api/handlers/node.go
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"runtime"
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -76,7 +75,6 @@ type healthCheckJSON struct {
 		InboundConns    int      `json:"inbound_conns"`
 		OutboundConns   int      `json:"outbound_conns"`
 		ListenAddresses []string `json:"p2p_listen_addresses"`
-		CPUCores        int      `json:"cpu_cores"`
 	} `json:"advanced"`
 }
 
@@ -173,9 +171,6 @@ func (h *Node) Health(w http.ResponseWriter, r *http.Request) error {
 	resp.ExecutionNode = healthStatus{h.NodeProber.CheckExecutionNodeHealth(ctx)}
 	resp.EventSyncer = healthStatus{(h.NodeProber.CheckEventSyncerHealth(ctx))}
 
-	// Report number of CPU cores.
-	resp.Advanced.CPUCores = runtime.NumCPU()
-
 	return api.Render(w, r, resp)
 }