Feat/add more metrics

api/modules.go

@@ -17,6 +17,7 @@ import (
 	"go.uber.org/fx"
 
 	"github.com/ipfs-force-community/sophon-messager/config"
+	"github.com/ipfs-force-community/sophon-messager/metrics"
 )
 
 var log = logging.Logger("api")
@@ -62,13 +63,16 @@ func RunAPI(lc fx.Lifecycle, localAuthCli *jwtclient.LocalAuthClient, remoteAuth
 		OnStart: func(ctx context.Context) error {
 			go func() {
 				log.Info("start rpcserver ", lst.Addr())
+				metrics.ApiState.Set(ctx, 1)
 				if err := apiserv.Serve(lst); err != nil && !errors.Is(err, http.ErrServerClosed) {
+					metrics.ApiState.Set(ctx, 0)
 					log.Errorf("start rpcserver failed: %v", err)
 				}
 			}()
 			return nil
 		},
 		OnStop: func(ctx context.Context) error {
+			defer metrics.ApiState.Set(ctx, 0)
 			return apiserv.Shutdown(ctx)
 		},
 	})

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/hunjixin/automapper v0.0.0-20191127090318-9b979ce72ce2
-	github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601
+	github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09
 	github.com/ipfs-force-community/sophon-auth v1.14.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/libp2p/go-libp2p v0.27.8

go.sum

@@ -513,8 +513,8 @@ github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
 github.com/ipfs-force-community/go-jsonrpc v0.1.9 h1:5QavBltfvV6fz/+EbYsCkVxJ1MSJncZm6YuPs1SLdZU=
 github.com/ipfs-force-community/go-jsonrpc v0.1.9/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM=
-github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601 h1:zxKQ30KAD6KfvSFAx9tuqQXLDsEHyF+eVaUBXXYC2bU=
-github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA=
+github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09 h1:qEI6ItxKtgOupMMuGJwqK5zEzztKKPUP1QKq9g+X5bM=
+github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA=
 github.com/ipfs-force-community/sophon-auth v1.14.0 h1:ctBJ6UHkcytEzfVPgiiHo0cW4FGQrE7r1H3Um0FcHbo=
 github.com/ipfs-force-community/sophon-auth v1.14.0/go.mod h1:d6J6u3zyIwcEajRho5BhVBcoIChEf0K76wP4yJEfEhc=
 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=

metrics/exporter.go

@@ -13,14 +13,6 @@ import (
 var log = logging.Logger("metric")
 
 func SetupMetrics(lc fx.Lifecycle, metricsConfig *metrics.MetricsConfig) error {
-	log.Infof("metrics config: enabled: %v, exporter type: %s, prometheus: %v, graphite: %v",
-		metricsConfig.Enabled, metricsConfig.Exporter.Type, metricsConfig.Exporter.Prometheus,
-		metricsConfig.Exporter.Graphite)
-
-	if !metricsConfig.Enabled {
-		return nil
-	}
-
 	if err := view.Register(
 		MessagerNodeViews...,
 	); err != nil {
@@ -29,26 +21,11 @@ func SetupMetrics(lc fx.Lifecycle, metricsConfig *metrics.MetricsConfig) error {
 
 	lc.Append(fx.Hook{
 		OnStart: func(ctx context.Context) error {
-			switch metricsConfig.Exporter.Type {
-			case metrics.ETPrometheus:
-				go func() {
-					if err := metrics.RegisterPrometheusExporter(ctx, metricsConfig.Exporter.Prometheus); err != nil {
-						log.Errorf("Register prometheus exporter err: %v", err)
-					}
-					log.Info("Prometheus exporter server graceful shutdown successful")
-				}()
-
-			case metrics.ETGraphite:
-				if err := metrics.RegisterGraphiteExporter(ctx, metricsConfig.Exporter.Graphite); err != nil {
-					log.Errorf("failed to register the exporter: %v", err)
-				}
-			default:
-				log.Warnf("invalid exporter type: %s", metricsConfig.Exporter.Type)
+			if metricsConfig != nil {
+				return metrics.SetupMetrics(ctx, metricsConfig)
 			}
-
 			return nil
 		},
 	})
-
 	return nil
 }

metrics/metrics.go

@@ -1,7 +1,9 @@
 package metrics
 
 import (
-	"github.com/filecoin-project/go-jsonrpc/metrics"
+	rpcMetrics "github.com/filecoin-project/go-jsonrpc/metrics"
+	"github.com/ipfs-force-community/metrics"
+
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
@@ -26,13 +28,17 @@ var (
 
 	NumOfMsgBlockedThreeMinutes = stats.Int64("blocked_three_minutes_msgs", "Number of messages blocked for more than 3 minutes", stats.UnitDimensionless)
 	NumOfMsgBlockedFiveMinutes  = stats.Int64("blocked_five_minutes_msgs", "Number of messages blocked for more than 5 minutes", stats.UnitDimensionless)
+	ChainHeadStableDelay        = stats.Int64("chain_head_stable_s", "Delay of chain head stabilization", stats.UnitSeconds)
+	ChainHeadStableDuration     = stats.Int64("chain_head_stable_dur_s", "Duration of chain head stabilization", stats.UnitSeconds)
+)
 
-	SelectedMsgNumOfLastRound = stats.Int64("selected_msg_num", "Number of selected messages in the last round", stats.UnitDimensionless)
-	ToPushMsgNumOfLastRound   = stats.Int64("topush_msg_num", "Number of to-push messages in the last round", stats.UnitDimensionless)
-	ErrMsgNumOfLastRound      = stats.Int64("err_msg_num", "Number of err messages in the last round", stats.UnitDimensionless)
+var (
+	SelectedMsgNumOfLastRound = metrics.NewInt64("selected_msg_num", "Number of selected messages in the last round", stats.UnitDimensionless, WalletAddress)
+	ToPushMsgNumOfLastRound   = metrics.NewInt64("topush_msg_num", "Number of to-push messages in the last round", stats.UnitDimensionless, WalletAddress)
+	ErrMsgNumOfLastRound      = metrics.NewInt64("err_msg_num", "Number of err messages in the last round", stats.UnitDimensionless, WalletAddress)
 
-	ChainHeadStableDelay    = stats.Int64("chain_head_stable_s", "Delay of chain head stabilization", stats.UnitSeconds)
-	ChainHeadStableDuration = stats.Int64("chain_head_stable_dur_s", "Duration of chain head stabilization", stats.UnitSeconds)
+	ApiState          = metrics.NewInt64("api/state", "api service state. 0: down, 1: up", "")
+	AddressNumInState = metrics.NewInt64WithCategory("address/num", "Number of addresses in the vary state", "")
 )
 
 var (
@@ -78,22 +84,6 @@ var (
 		TagKeys:     []tag.Key{WalletAddress},
 	}
 
-	SelectedMsgNumOfLastRoundView = &view.View{
-		Measure:     SelectedMsgNumOfLastRound,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{WalletAddress},
-	}
-	ToPushMsgNumOfLastRoundView = &view.View{
-		Measure:     ToPushMsgNumOfLastRound,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{WalletAddress},
-	}
-	ErrMsgNumOfLastRoundView = &view.View{
-		Measure:     ErrMsgNumOfLastRound,
-		Aggregation: view.LastValue(),
-		TagKeys:     []tag.Key{WalletAddress},
-	}
-
 	ChainHeadStableDelayView = &view.View{
 		Measure:     ChainHeadStableDelay,
 		Aggregation: view.LastValue(),
@@ -116,10 +106,6 @@ var MessagerNodeViews = append([]*view.View{
 	NumOfMsgBlockedThreeMinutesView,
 	NumOfMsgBlockedFiveMinutesView,
 
-	SelectedMsgNumOfLastRoundView,
-	ToPushMsgNumOfLastRoundView,
-	ErrMsgNumOfLastRoundView,
-
 	ChainHeadStableDelayView,
 	ChainHeadStableDurationView,
-}, metrics.DefaultViews...)
+}, rpcMetrics.DefaultViews...)

service/message_selector.go

@@ -18,7 +18,6 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	"go.opencensus.io/stats"
 	"go.opencensus.io/tag"
 	"go.uber.org/zap"
 	"modernc.org/mathutil"
@@ -210,9 +209,9 @@ func addrSelectMsgNum(addrList []*types.Address, defSelMsgNum uint64) map[addres
 func recordMetric(ctx context.Context, addr address.Address, selectResult *MsgSelectResult) {
 	ctx, _ = tag.New(ctx, tag.Upsert(metrics.WalletAddress, addr.String()))
 
-	stats.Record(ctx, metrics.SelectedMsgNumOfLastRound.M(int64(len(selectResult.SelectMsg))))
-	stats.Record(ctx, metrics.ToPushMsgNumOfLastRound.M(int64(len(selectResult.ToPushMsg))))
-	stats.Record(ctx, metrics.ErrMsgNumOfLastRound.M(int64(len(selectResult.ErrMsg))))
+	metrics.SelectedMsgNumOfLastRound.Set(ctx, int64(len(selectResult.SelectMsg)))
+	metrics.ToPushMsgNumOfLastRound.Set(ctx, int64(len(selectResult.ToPushMsg)))
+	metrics.ErrMsgNumOfLastRound.Set(ctx, int64(len(selectResult.ErrMsg)))
 }
 
 var errSingMessage = errors.New("sign message failed")

service/message_service.go

@@ -1062,6 +1062,22 @@ func (ms *MessageService) recordMetricsProc(ctx context.Context) {
 			} else {
 				stats.Record(ctx, metrics.NumOfFailedMsg.M(int64(len(msgs))))
 			}
+
+			allAddrs, err := ms.repo.AddressRepo().ListAddress(ctx)
+			if err != nil {
+				log.Errorf("get all address err: %s", err)
+			} else {
+				addrInSate := map[types.AddressState]int64{}
+				for _, addr := range allAddrs {
+					if _, ok := addrInSate[addr.State]; !ok {
+						addrInSate[addr.State] = 0
+					}
+					addrInSate[addr.State]++
+				}
+				for state, count := range addrInSate {
+					metrics.AddressNumInState.Set(ctx, state.String(), count)
+				}
+			}
 		}
 	}
 }