diff --git a/td2/dashboard/server.go b/td2/dashboard/server.go index 9f0eba5..80cc8b0 100644 --- a/td2/dashboard/server.go +++ b/td2/dashboard/server.go @@ -3,8 +3,6 @@ package dash import ( "embed" "encoding/json" - "github.com/gorilla/websocket" - "github.com/textileio/go-threads/broadcast" "io/fs" "log" "net/http" @@ -12,6 +10,9 @@ import ( "sort" "sync" "time" + + "github.com/gorilla/websocket" + "github.com/textileio/go-threads/broadcast" ) var ( diff --git a/td2/dashboard/types.go b/td2/dashboard/types.go index 7e98cb4..cb96b03 100644 --- a/td2/dashboard/types.go +++ b/td2/dashboard/types.go @@ -1,20 +1,21 @@ package dash type ChainStatus struct { - MsgType string `json:"msgType"` - Name string `json:"name"` - ChainId string `json:"chain_id"` - Moniker string `json:"moniker"` - Bonded bool `json:"bonded"` - Jailed bool `json:"jailed"` - Tombstoned bool `json:"tombstoned"` - Missed int64 `json:"missed"` - Window int64 `json:"window"` - Nodes int `json:"nodes"` - HealthyNodes int `json:"healthy_nodes"` - ActiveAlerts int `json:"active_alerts"` - Height int64 `json:"height"` - LastError string `json:"last_error"` + MsgType string `json:"msgType"` + Name string `json:"name"` + ChainId string `json:"chain_id"` + Moniker string `json:"moniker"` + Bonded bool `json:"bonded"` + Jailed bool `json:"jailed"` + Tombstoned bool `json:"tombstoned"` + Missed int64 `json:"missed"` + Window int64 `json:"window"` + MinSignedPerWindow float64 `json:"min_signed_per_window"` + Nodes int `json:"nodes"` + HealthyNodes int `json:"healthy_nodes"` + ActiveAlerts int `json:"active_alerts"` + Height int64 `json:"height"` + LastError string `json:"last_error"` Blocks []int `json:"blocks"` } diff --git a/td2/rpc.go b/td2/rpc.go index 5f61687..f25ac9a 100644 --- a/td2/rpc.go +++ b/td2/rpc.go @@ -24,6 +24,7 @@ func (cc *ChainConfig) newRpc() error { for _, endpoint := range cc.Nodes { anyWorking = anyWorking || !endpoint.down } + // grab the first working endpoint tryUrl := func(u string) (msg string, down, syncing bool) { _, err := url.Parse(u) @@ -106,21 +107,22 @@ func (cc *ChainConfig) newRpc() error { cc.lastError = "no usable RPC endpoints available for " + cc.ChainId if td.EnableDash { td.updateChan <- &dash.ChainStatus{ - MsgType: "status", - Name: cc.name, - ChainId: cc.ChainId, - Moniker: cc.valInfo.Moniker, - Bonded: cc.valInfo.Bonded, - Jailed: cc.valInfo.Jailed, - Tombstoned: cc.valInfo.Tombstoned, - Missed: cc.valInfo.Missed, - Window: cc.valInfo.Window, - Nodes: len(cc.Nodes), - HealthyNodes: 0, - ActiveAlerts: 1, - Height: 0, - LastError: cc.lastError, - Blocks: cc.blocksResults, + MsgType: "status", + Name: cc.name, + ChainId: cc.ChainId, + Moniker: cc.valInfo.Moniker, + Bonded: cc.valInfo.Bonded, + Jailed: cc.valInfo.Jailed, + Tombstoned: cc.valInfo.Tombstoned, + Missed: cc.valInfo.Missed, + Window: cc.valInfo.Window, + MinSignedPerWindow: cc.minSignedPerWindow, + Nodes: len(cc.Nodes), + HealthyNodes: 0, + ActiveAlerts: 1, + Height: 0, + LastError: cc.lastError, + Blocks: cc.blocksResults, } } return errors.New("no usable endpoints available for " + cc.ChainId) diff --git a/td2/run.go b/td2/run.go index 0cce9fa..c98c6f5 100644 --- a/td2/run.go +++ b/td2/run.go @@ -107,6 +107,12 @@ func Run(configFile, stateFile, chainConfigDirectory string, password *string) e time.Sleep(5 * time.Second) continue } + + e = cc.GetMinSignedPerWindow() + if e != nil { + l("🛑", cc.ChainId, e) + } + e = cc.GetValInfo(true) if e != nil { l("🛑", cc.ChainId, e) diff --git a/td2/static/index.html b/td2/static/index.html index 68dd61d..fdffb8c 100644 --- a/td2/static/index.html +++ b/td2/static/index.html @@ -42,6 +42,7 @@ Moniker Bonded Uptime + Threshold RPC Nodes diff --git a/td2/static/status.js b/td2/static/status.js index e18ef48..5d6c750 100644 --- a/td2/static/status.js +++ b/td2/static/status.js @@ -107,6 +107,9 @@ function updateTable(status) { window += `${(100 - (status.Status[i].missed / status.Status[i].window) * 100).toFixed(2)}%` } window += `
${_.escape(status.Status[i].missed)} / ${_.escape(status.Status[i].window)}
` + + let threshold = "" + threshold += `${100 * status.Status[i].min_signed_per_window}%`; let nodes = `${_.escape(status.Status[i].healthy_nodes)} / ${_.escape(status.Status[i].nodes)}` if (status.Status[i].healthy_nodes < status.Status[i].nodes) { @@ -131,7 +134,8 @@ function updateTable(status) { } r.insertCell(4).innerHTML = `
${bonded}
` r.insertCell(5).innerHTML = `
${window}
` - r.insertCell(6).innerHTML = `
${nodes}
` + r.insertCell(6).innerHTML = `
${threshold}
` + r.insertCell(7).innerHTML = `
${nodes}
` } } diff --git a/td2/types.go b/td2/types.go index 8012949..a36840e 100644 --- a/td2/types.go +++ b/td2/types.go @@ -83,18 +83,19 @@ type savedState struct { // ChainConfig represents a validator to be monitored on a chain, it is somewhat of a misnomer since multiple // validators can be monitored on a single chain. type ChainConfig struct { - name string - wsclient *TmConn // custom websocket client to work around wss:// bugs in tendermint - client *rpchttp.HTTP // legit tendermint client - noNodes bool // tracks if all nodes are down - valInfo *ValInfo // recent validator state, only refreshed every few minutes - lastValInfo *ValInfo // use for detecting newly-jailed/tombstone - blocksResults []int - lastError string - lastBlockTime time.Time - lastBlockAlarm bool - lastBlockNum int64 - activeAlerts int + name string + wsclient *TmConn // custom websocket client to work around wss:// bugs in tendermint + client *rpchttp.HTTP // legit tendermint client + noNodes bool // tracks if all nodes are down + valInfo *ValInfo // recent validator state, only refreshed every few minutes + lastValInfo *ValInfo // use for detecting newly-jailed/tombstone + minSignedPerWindow float64 // instantly see the validator risk level + blocksResults []int + lastError string + lastBlockTime time.Time + lastBlockAlarm bool + lastBlockNum int64 + activeAlerts int statTotalSigns float64 statTotalProps float64 @@ -326,19 +327,20 @@ func validateConfig(c *Config) (fatal bool, problems []string) { } if td.EnableDash { td.updateChan <- &dash.ChainStatus{ - MsgType: "status", - Name: v.name, - ChainId: v.ChainId, - Moniker: v.valInfo.Moniker, - Bonded: v.valInfo.Bonded, - Jailed: v.valInfo.Jailed, - Tombstoned: v.valInfo.Tombstoned, - Missed: v.valInfo.Missed, - Window: v.valInfo.Window, - Nodes: len(v.Nodes), - HealthyNodes: 0, - ActiveAlerts: 0, - Blocks: v.blocksResults, + MsgType: "status", + Name: v.name, + ChainId: v.ChainId, + Moniker: v.valInfo.Moniker, + Bonded: v.valInfo.Bonded, + Jailed: v.valInfo.Jailed, + Tombstoned: v.valInfo.Tombstoned, + Missed: v.valInfo.Missed, + MinSignedPerWindow: v.minSignedPerWindow, + Window: v.valInfo.Window, + Nodes: len(v.Nodes), + HealthyNodes: 0, + ActiveAlerts: 0, + Blocks: v.blocksResults, } } } diff --git a/td2/validator.go b/td2/validator.go index ac01ce7..ad00588 100644 --- a/td2/validator.go +++ b/td2/validator.go @@ -28,6 +28,37 @@ type ValInfo struct { Valcons string `json:"valcons"` } +// GetMinSignedPerWindow The check the minimum signed threshold of the validator. +func (cc *ChainConfig) GetMinSignedPerWindow() (err error) { + if cc.client == nil { + return errors.New("nil rpc client") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + qParams := &slashing.QueryParamsRequest{} + b, err := qParams.Marshal() + if err != nil { + return + } + resp, err := cc.client.ABCIQuery(ctx, "/cosmos.slashing.v1beta1.Query/Params", b) + if err != nil { + return + } + if resp.Response.Value == nil { + err = errors.New("🛑 could not query slashing params, got empty response") + return + } + params := &slashing.QueryParamsResponse{} + err = params.Unmarshal(resp.Response.Value) + if err != nil { + return + } + + cc.minSignedPerWindow = params.Params.MinSignedPerWindow.MustFloat64() + return +} + // GetValInfo the first bool is used to determine if extra information about the validator should be printed. func (cc *ChainConfig) GetValInfo(first bool) (err error) { if cc.client == nil { diff --git a/td2/ws.go b/td2/ws.go index ce1ef4d..f3d83e8 100644 --- a/td2/ws.go +++ b/td2/ws.go @@ -6,14 +6,15 @@ import ( "encoding/json" "errors" "fmt" - dash "github.com/blockpane/tenderduty/v2/td2/dashboard" - "github.com/gorilla/websocket" - pbtypes "github.com/tendermint/tendermint/proto/tendermint/types" "log" "net/url" "strconv" "strings" "time" + + dash "github.com/blockpane/tenderduty/v2/td2/dashboard" + "github.com/gorilla/websocket" + pbtypes "github.com/tendermint/tendermint/proto/tendermint/types" ) const ( @@ -129,6 +130,7 @@ func (cc *ChainConfig) WsRun() { cc.lastError = time.Now().UTC().String() + " " + info l(warn) } + switch signState { case Statusmissed: cc.statTotalMiss += 1 @@ -164,24 +166,26 @@ func (cc *ChainConfig) WsRun() { case cc.valInfo.Jailed: info += "- validator is jailed\n" } + cc.activeAlerts = alarms.getCount(cc.name) if td.EnableDash { td.updateChan <- &dash.ChainStatus{ - MsgType: "status", - Name: cc.name, - ChainId: cc.ChainId, - Moniker: cc.valInfo.Moniker, - Bonded: cc.valInfo.Bonded, - Jailed: cc.valInfo.Jailed, - Tombstoned: cc.valInfo.Tombstoned, - Missed: cc.valInfo.Missed, - Window: cc.valInfo.Window, - Nodes: len(cc.Nodes), - HealthyNodes: healthyNodes, - ActiveAlerts: cc.activeAlerts, - Height: update.Height, - LastError: info, - Blocks: cc.blocksResults, + MsgType: "status", + Name: cc.name, + ChainId: cc.ChainId, + Moniker: cc.valInfo.Moniker, + Bonded: cc.valInfo.Bonded, + Jailed: cc.valInfo.Jailed, + Tombstoned: cc.valInfo.Tombstoned, + Missed: cc.valInfo.Missed, + Window: cc.valInfo.Window, + MinSignedPerWindow: cc.minSignedPerWindow, + Nodes: len(cc.Nodes), + HealthyNodes: healthyNodes, + ActiveAlerts: cc.activeAlerts, + Height: update.Height, + LastError: info, + Blocks: cc.blocksResults, } }