feat: sp service adds metrics (#239)

Co-authored-by: DylanYong <dylan.y@nodereal.io>
bnb-chain · Mar 28, 2023 · 812a9ea · 812a9ea
1 parent f1a2b2e
commit 812a9ea
Show file tree

Hide file tree

Showing 19 changed files with 251 additions and 144 deletions.
diff --git a/config/config_template.toml b/config/config_template.toml
@@ -60,14 +60,14 @@ Modules = ["epoch", "bucket", "object", "payment"]
 Dsn = "localhost:3308"
 
 [P2PCfg]
-ListenAddress = "127.0.0.1:9933"
+ListenAddress = "127.0.0.1:9833"
 P2PPrivateKey = ""
 Bootstrap = []
 PingPeriod = 2
 
 [MetricsCfg]
 Enabled = false
-HTTPAddress = "localhost:9833"
+HTTPAddress = "localhost:24036"
 
 [LogCfg]
 Level = "debug"

diff --git a/pkg/middleware/grpc/metrics.go b/pkg/middleware/grpc/metrics.go
@@ -0,0 +1,39 @@
+package grpc
+
+import (
+	"runtime/debug"
+
+	openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2"
+	grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+)
+
+var (
+	gRPCPanicRecoveryHandler = func(p interface{}) (err error) {
+		metrics.PanicsTotal.WithLabelValues().Inc()
+		log.Errorw("recovered from panic", "panic", p, "stack", debug.Stack())
+		return status.Errorf(codes.Internal, "%s", p)
+	}
+)
+
+// GetDefaultServerInterceptor returns default gRPC server interceptor
+func GetDefaultServerInterceptor() []grpc.ServerOption {
+	options := []grpc.ServerOption{}
+	options = append(options, grpc.ChainUnaryInterceptor(openmetrics.UnaryServerInterceptor(metrics.DefaultGRPCServerMetrics),
+		grpcrecovery.UnaryServerInterceptor(grpcrecovery.WithRecoveryHandler(gRPCPanicRecoveryHandler))))
+	options = append(options, grpc.ChainStreamInterceptor(openmetrics.StreamServerInterceptor(metrics.DefaultGRPCServerMetrics)))
+	return options
+}
+
+// GetDefaultClientInterceptor returns default gRPC client interceptor
+func GetDefaultClientInterceptor() []grpc.DialOption {
+	options := []grpc.DialOption{}
+	options = append(options, grpc.WithChainUnaryInterceptor(openmetrics.UnaryClientInterceptor(metrics.DefaultGRPCClientMetrics)))
+	options = append(options, grpc.WithChainStreamInterceptor(openmetrics.StreamClientInterceptor(metrics.DefaultGRPCClientMetrics)))
+	return options
+}
diff --git a/pkg/p2p/node.go b/pkg/p2p/node.go
@@ -5,12 +5,12 @@ import (
 	"strings"
 	"time"
 
-	signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
+	storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
 	ggio "github.com/gogo/protobuf/io"
 	"github.com/gogo/protobuf/proto"
 	ds "github.com/ipfs/go-datastore"
 	leveldb "github.com/ipfs/go-ds-leveldb"
-	libp2p "github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/peerstore"
@@ -19,7 +19,7 @@ import (
 
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/p2p/types"
-	storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
+	signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
 )
 
 // Node defines the p2p protocol node, encapsulates the go-lib.p2p
@@ -102,7 +102,7 @@ func (n *Node) Name() string {
 
 // Start runs background task that trigger broadcast ping request
 func (n *Node) Start(ctx context.Context) error {
-	go n.eventloop()
+	go n.eventLoop()
 	return nil
 }
 
@@ -168,8 +168,8 @@ func (n *Node) GetApproval(object *storagetypes.ObjectInfo, expectedAccept int,
 	}
 }
 
-// eventloop run the background task
-func (n *Node) eventloop() {
+// eventLoop run the background task
+func (n *Node) eventLoop() {
 	ticker := time.NewTicker(time.Duration(n.config.PingPeriod) * time.Second)
 	for {
 		select {

diff --git a/service/challenge/challenge.go b/service/challenge/challenge.go
@@ -4,16 +4,19 @@ import (
 	"context"
 	"net"
 
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 
 	"github.com/bnb-chain/greenfield-storage-provider/model"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
 	"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
 	psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
 	"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
+	utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
 )
 
 var _ lifecycle.Service = &Challenge{}
@@ -25,6 +28,7 @@ type Challenge struct {
 	spDB       sqldb.SPDB
 	rcScope    rcmgr.ResourceScope
 	pieceStore *psclient.StoreClient
+	grpcServer *grpc.Server
 }
 
 // NewChallengeService returns an instance of Challenge that implementation of
@@ -62,29 +66,35 @@ func (challenge *Challenge) Name() string {
 // Start the challenge gRPC service
 func (challenge *Challenge) Start(ctx context.Context) error {
 	errCh := make(chan error)
-
-	go func(errCh chan error) {
-		lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
-		errCh <- err
-		if err != nil {
-			log.Errorw("failed to listen", "error", err)
-			return
-		}
-		grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
-		types.RegisterChallengeServiceServer(grpcServer, challenge)
-		reflection.Register(grpcServer)
-		if err = grpcServer.Serve(lis); err != nil {
-			log.Errorw("failed to serve", "error", err)
-			return
-		}
-	}(errCh)
-
+	go challenge.serve(errCh)
 	err := <-errCh
 	return err
 }
 
 // Stop the challenge gRPC service and recycle the resources
 func (challenge *Challenge) Stop(ctx context.Context) error {
+	challenge.grpcServer.GracefulStop()
 	challenge.rcScope.Release()
 	return nil
 }
+
+func (challenge *Challenge) serve(errCh chan error) {
+	lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
+	errCh <- err
+	if err != nil {
+		log.Errorw("failed to listen", "error", err)
+		return
+	}
+
+	options := utilgrpc.GetDefaultServerOptions()
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
+	}
+	challenge.grpcServer = grpc.NewServer(options...)
+	types.RegisterChallengeServiceServer(challenge.grpcServer, challenge)
+	reflection.Register(challenge.grpcServer)
+	if err = challenge.grpcServer.Serve(lis); err != nil {
+		log.Errorw("failed to serve", "error", err)
+		return
+	}
+}
diff --git a/service/challenge/client/challenge_client.go b/service/challenge/client/challenge_client.go
@@ -3,13 +3,14 @@ package client
 import (
 	"context"
 
-	"github.com/bnb-chain/greenfield-storage-provider/model"
-	merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 
+	merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
 	"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
+	utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
 )
 
 // ChallengeClient is a challenge gRPC service client wrapper
@@ -21,10 +22,11 @@ type ChallengeClient struct {
 
 // NewChallengeClient return a ChallengeClient instance
 func NewChallengeClient(address string) (*ChallengeClient, error) {
-	conn, err := grpc.DialContext(context.Background(), address,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
-		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
+	options := utilgrpc.GetDefaultClientOptions()
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
+	}
+	conn, err := grpc.DialContext(context.Background(), address, options...)
 	if err != nil {
 		log.Errorw("failed to dial challenge", "error", err)
 		return nil, err

diff --git a/service/downloader/client/downloader_client.go b/service/downloader/client/downloader_client.go
@@ -3,13 +3,14 @@ package client
 import (
 	"context"
 
-	"github.com/bnb-chain/greenfield-storage-provider/model"
+	storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
 	"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
-	storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
+	utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
 )
 
 // DownloaderClient is a downloader gRPC service client wrapper
@@ -21,10 +22,11 @@ type DownloaderClient struct {
 
 // NewDownloaderClient returns a DownloaderClient instance
 func NewDownloaderClient(address string) (*DownloaderClient, error) {
-	conn, err := grpc.DialContext(context.Background(), address,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
-		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
+	options := utilgrpc.GetDefaultClientOptions()
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
+	}
+	conn, err := grpc.DialContext(context.Background(), address, options...)
 	if err != nil {
 		log.Errorw("failed to dial downloader", "error", err)
 		return nil, err

diff --git a/service/downloader/downloader.go b/service/downloader/downloader.go
@@ -4,16 +4,19 @@ import (
 	"context"
 	"net"
 
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
-	"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 
 	"github.com/bnb-chain/greenfield-storage-provider/model"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
 	"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
 	psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
+	"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
+	utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
 )
 
 var _ lifecycle.Service = &Downloader{}
@@ -24,6 +27,7 @@ type Downloader struct {
 	config     *DownloaderConfig
 	spDB       sqldb.SPDB
 	pieceStore *psclient.StoreClient
+	grpcServer *grpc.Server
 	rcScope    rcmgr.ResourceScope
 }
 
@@ -60,29 +64,35 @@ func (downloader *Downloader) Name() string {
 // Start the downloader gRPC service
 func (downloader *Downloader) Start(ctx context.Context) error {
 	errCh := make(chan error)
-
-	go func(errCh chan error) {
-		lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
-		errCh <- err
-		if err != nil {
-			log.Errorw("failed to listen", "error", err)
-			return
-		}
-		grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
-		types.RegisterDownloaderServiceServer(grpcServer, downloader)
-		reflection.Register(grpcServer)
-		if err = grpcServer.Serve(lis); err != nil {
-			log.Errorw("failed to serve", "error", err)
-			return
-		}
-	}(errCh)
-
+	go downloader.serve(errCh)
 	err := <-errCh
 	return err
 }
 
 // Stop the downloader gRPC service and recycle the resources
 func (downloader *Downloader) Stop(ctx context.Context) error {
+	downloader.grpcServer.GracefulStop()
 	downloader.rcScope.Release()
 	return nil
 }
+
+func (downloader *Downloader) serve(errCh chan error) {
+	lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
+	errCh <- err
+	if err != nil {
+		log.Errorw("failed to listen", "error", err)
+		return
+	}
+
+	options := utilgrpc.GetDefaultServerOptions()
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
+	}
+	downloader.grpcServer = grpc.NewServer(options...)
+	types.RegisterDownloaderServiceServer(downloader.grpcServer, downloader)
+	reflection.Register(downloader.grpcServer)
+	if err = downloader.grpcServer.Serve(lis); err != nil {
+		log.Errorw("failed to serve", "error", err)
+		return
+	}
+}
diff --git a/service/p2p/client/p2p_client.go b/service/p2p/client/p2p_client.go
@@ -7,8 +7,9 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
-	"github.com/bnb-chain/greenfield-storage-provider/model"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
 	p2ptypes "github.com/bnb-chain/greenfield-storage-provider/pkg/p2p/types"
 	"github.com/bnb-chain/greenfield-storage-provider/service/p2p/types"
 )
@@ -22,10 +23,12 @@ type P2PClient struct {
 
 // NewP2PClient return a P2PClient instance
 func NewP2PClient(address string) (*P2PClient, error) {
-	conn, err := grpc.DialContext(context.Background(), address,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
-		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
+	options := []grpc.DialOption{}
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
+	}
+	options = append(options, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.DialContext(context.Background(), address, options...)
 	if err != nil {
 		log.Errorw("failed to dial p2p server", "error", err)
 		return nil, err

diff --git a/service/p2p/p2p.go b/service/p2p/p2p.go
@@ -14,6 +14,8 @@ import (
 	"github.com/bnb-chain/greenfield-storage-provider/model"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
+	mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/p2p"
 	p2ptypes "github.com/bnb-chain/greenfield-storage-provider/service/p2p/types"
 	signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
@@ -92,11 +94,14 @@ func (p *P2PServer) serve(errCh chan error) {
 		return
 	}
 
-	grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize))
-	p2ptypes.RegisterP2PServiceServer(grpcServer, p)
-	p.grpcServer = grpcServer
-	reflection.Register(grpcServer)
-	if err := grpcServer.Serve(lis); err != nil {
+	options := []grpc.ServerOption{}
+	if metrics.GetMetrics().Enabled() {
+		options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
+	}
+	p.grpcServer = grpc.NewServer(options...)
+	p2ptypes.RegisterP2PServiceServer(p.grpcServer, p)
+	reflection.Register(p.grpcServer)
+	if err := p.grpcServer.Serve(lis); err != nil {
 		log.Errorw("failed to start grpc server", "err", err)
 		return
 	}