CLI: extend 'show cluster' - add 'alert' column

* move version & build to the summary - show version (or build) with individual nodes _iff_ there are different versions and builds, respectively * add alert column; hide it _iff_ all state flags are OK - for enumeration, see cmn/cos/node_state_flags Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
NVIDIA · Aug 21, 2024 · 40d6580 · 40d6580
1 parent 5c94da9
commit 40d6580
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 9 deletions.
diff --git a/ais/htrun.go b/ais/htrun.go
@@ -1134,6 +1134,9 @@ func (h *htrun) statsAndStatus() (ds *stats.NodeStatus) {
 		Node: stats.Node{
 			Snode: h.si,
 		},
+		Cluster: cos.NodeStateInfo{
+			Flags: cos.NodeStateFlags(h.statsT.Get(stats.NodeStateFlags)),
+		},
 		SmapVersion:    smap.Version,
 		MemCPUInfo:     apc.GetMemCPU(),
 		DeploymentType: deploymentType(),

diff --git a/cmd/cli/cli/daeclu.go b/cmd/cli/cli/daeclu.go
@@ -65,8 +65,9 @@ func cluDaeStatus(c *cli.Context, smap *meta.Smap, tstatusMap, pstatusMap teb.St
 	tableP := teb.NewDaeMapStatus(&body.Status, smap, apc.Proxy, units)
 	tableT := teb.NewDaeMapStatus(&body.Status, smap, apc.Target, units)
 
-	// total num disks and capacity
+	// totals: num disks and capacity; software version and build tiume
 	body.NumDisks, body.Capacity = _totals(body.Status.Tmap, units, cfg)
+	body.Version, body.BuildTime = _clusoft(body.Status.Tmap, body.Status.Pmap)
 
 	out := tableP.Template(false) + "\n"
 	out += tableT.Template(false) + "\n"
@@ -133,3 +134,28 @@ outer:
 
 	return num, cs
 }
+
+func _clusoft(nodemaps ...teb.StstMap) (version, build string) {
+	var multiver, multibuild bool
+	for _, m := range nodemaps {
+		for _, ds := range m {
+			if !multiver {
+				if version == "" {
+					version = ds.Version
+				} else if version != ds.Version {
+					multiver = true
+					version = ""
+				}
+			}
+			if !multibuild {
+				if build == "" {
+					build = ds.BuildTime
+				} else if build != ds.BuildTime {
+					multibuild = true
+					build = ""
+				}
+			}
+		}
+	}
+	return version, build
+}
diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.22.3
 
 require (
-	github.com/NVIDIA/aistore v1.3.24-0.20240819221047-053ce1175e42
+	github.com/NVIDIA/aistore v1.3.24-0.20240821144511-7324344cd3cd
 	github.com/fatih/color v1.17.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.20.0

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.24-0.20240819221047-053ce1175e42 h1:JMzCMPLftDSGa6/TIXlHFCJCfLvPl+a20gf3ZY98OL4=
-github.com/NVIDIA/aistore v1.3.24-0.20240819221047-053ce1175e42/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM=
+github.com/NVIDIA/aistore v1.3.24-0.20240821144511-7324344cd3cd h1:k619Yjgc8Rsei9yRiG4WwW+xrj8RstITaVtNPfuqB0c=
+github.com/NVIDIA/aistore v1.3.24-0.20240821144511-7324344cd3cd/go.mod h1:si83S9r29vwIC0f0CE2Mk+25bFiaN6mmVlmuBpP4hHM=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/cmd/cli/teb/daeclu.go b/cmd/cli/teb/daeclu.go
@@ -24,10 +24,12 @@ const (
 	colLoadAvg   = "LOAD AVERAGE"
 	colRebalance = "REBALANCE"
 	colUptime    = "UPTIME"
+	colPodName   = "K8s POD"
 	colStatus    = "STATUS"
 	colVersion   = "VERSION"
 	colBuildTime = "BUILD TIME"
-	colPodName   = "K8s POD"
+
+	colStateFlags = "ALERT"
 )
 
 // TODO: extend api.GetClusterSysInfo() and api.GetStatsAndStatus to return memsys.Pressure
@@ -70,6 +72,7 @@ func newTableProxies(ps StstMap, smap *meta.Smap, units string) *Table {
 		pods     = h.pods()
 		status   = h.onlineStatus()
 		versions = h.versions()
+		builds   = h.buildTimes()
 		cols     = []*header{
 			{name: colProxy},
 			{name: colMemUsed},
@@ -78,8 +81,9 @@ func newTableProxies(ps StstMap, smap *meta.Smap, units string) *Table {
 			{name: colUptime},
 			{name: colPodName, hide: len(pods) == 1 && pods[0] == ""},
 			{name: colStatus, hide: len(status) == 1 && status[0] == NodeOnline},
-			{name: colVersion, hide: len(versions) == 1 && len(ps) > 1},
-			{name: colBuildTime, hide: len(versions) == 1 && len(ps) > 1}, // intended
+			{name: colVersion, hide: len(versions) == 1 && len(builds) == 1},
+			{name: colBuildTime, hide: len(versions) == 1 && len(builds) == 1},
+			{name: colStateFlags, hide: ps.allStateFlagsOK()},
 		}
 		table = newTable(cols...)
 	)
@@ -130,6 +134,7 @@ func newTableProxies(ps StstMap, smap *meta.Smap, units string) *Table {
 			ds.Status,
 			ds.Version,
 			ds.BuildTime,
+			ds.Cluster.Flags.String(),
 		}
 		table.addRow(row)
 	}
@@ -170,6 +175,7 @@ func newTableTargets(ts StstMap, smap *meta.Smap, units string) *Table {
 		pods     = h.pods()
 		status   = h.onlineStatus()
 		versions = h.versions()
+		builds   = h.buildTimes()
 		cols     = []*header{
 			{name: colTarget},
 			{name: colMemUsed},
@@ -181,8 +187,9 @@ func newTableTargets(ts StstMap, smap *meta.Smap, units string) *Table {
 			{name: colUptime},
 			{name: colPodName, hide: len(pods) == 1 && pods[0] == ""},
 			{name: colStatus, hide: len(status) == 1 && status[0] == NodeOnline},
-			{name: colVersion, hide: len(versions) == 1 && len(ts) > 1},
-			{name: colBuildTime, hide: len(versions) == 1 && len(ts) > 1}, // intended
+			{name: colVersion, hide: len(versions) == 1 && len(builds) == 1},
+			{name: colBuildTime, hide: len(versions) == 1 && len(builds) == 1},
+			{name: colStateFlags, hide: ts.allStateFlagsOK()},
 		}
 		table = newTable(cols...)
 	)
@@ -244,6 +251,7 @@ func newTableTargets(ts StstMap, smap *meta.Smap, units string) *Table {
 			ds.Status,
 			ds.Version,
 			ds.BuildTime,
+			ds.Cluster.Flags.String(),
 		}
 		table.addRow(row)
 	}

diff --git a/cmd/cli/teb/templates.go b/cmd/cli/teb/templates.go
@@ -52,6 +52,7 @@ const (
 		indent1 + "Targets:\t{{FormatTargetsSumm .Smap .NumDisks}}\n" +
 		indent1 + "Capacity:\t{{.Capacity}}\n" +
 		indent1 + "Cluster Map:\t{{FormatSmap .Smap}}\n" +
+		indent1 + "Software:\t{{FormatCluSoft .Version .BuildTime}}\n" +
 		indent1 + "Deployment:\t{{ ( Deployments .Status) }}\n" +
 		indent1 + "Status:\t{{ ( OnlineStatus .Status) }}\n" +
 		indent1 + "Rebalance:\t{{ ( Rebalance .Status) }}\n" +
@@ -366,6 +367,8 @@ type (
 		CluConfig *cmn.ClusterConfig
 		Status    StatsAndStatusHelper
 		Capacity  string
+		Version   string // when all equal
+		BuildTime string // ditto
 		NumDisks  int
 	}
 	ListBucketsHelper struct {
@@ -397,6 +400,7 @@ var (
 		"FormatObjCustom":     fmtObjCustom,
 		"FormatDaemonID":      fmtDaemonID,
 		"FormatSmap":          fmtSmap,
+		"FormatCluSoft":       fmtCluSoft,
 		"FormatProxiesSumm":   fmtProxiesSumm,
 		"FormatTargetsSumm":   fmtTargetsSumm,
 		"FormatCapPctMAM":     fmtCapPctMAM,
@@ -530,3 +534,12 @@ func (h *StatsAndStatusHelper) toSlice(jtag string) []string {
 	}
 	return res
 }
+
+func (m StstMap) allStateFlagsOK() bool {
+	for _, ds := range m {
+		if !ds.Cluster.Flags.IsOK() {
+			return false
+		}
+	}
+	return true
+}
diff --git a/cmd/cli/teb/utils.go b/cmd/cli/teb/utils.go
@@ -177,6 +177,16 @@ func fmtSmap(smap *meta.Smap) string {
 	return fmt.Sprintf("version %d, UUID %s, primary %s", smap.Version, smap.UUID, smap.Primary.StringEx())
 }
 
+func fmtCluSoft(version, build string) string {
+	if version == "" {
+		return unknownVal
+	}
+	if build == "" {
+		return version + " (build: " + unknownVal + ")"
+	}
+	return version + " (build: " + build + ")"
+}
+
 func fmtStringList(lst []string) string {
 	if len(lst) == 0 {
 		return unknownVal

diff --git a/cmn/cos/node_state_info.go → cmn/cos/node_state_flags.go b/cmn/cos/node_state_info.go → cmn/cos/node_state_flags.go