Skip to content

Commit

Permalink
feat: sp service adds metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
yzhaoyu committed Mar 27, 2023
1 parent 6fcfe19 commit 0f00a69
Show file tree
Hide file tree
Showing 12 changed files with 170 additions and 90 deletions.
4 changes: 2 additions & 2 deletions config/config_template.toml
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,14 @@ Modules = ["epoch", "bucket", "object", "payment"]
Dsn = "localhost:3308"

[P2PCfg]
ListenAddress = "127.0.0.1:9933"
ListenAddress = "127.0.0.1:9833"
P2PPrivateKey = ""
Bootstrap = []
PingPeriod = 2

[MetricsCfg]
Enabled = false
HTTPAddress = "localhost:9833"
HTTPAddress = "localhost:24036"

[LogCfg]
Level = "debug"
Expand Down
39 changes: 39 additions & 0 deletions pkg/middleware/grpc/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package grpc

import (
"runtime/debug"

openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2"
grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
)

var (
gRPCPanicRecoveryHandler = func(p interface{}) (err error) {
metrics.PanicsTotal.WithLabelValues().Inc()
log.Errorw("recovered from panic", "panic", p, "stack", debug.Stack())
return status.Errorf(codes.Internal, "%s", p)
}
)

// GetDefaultServerInterceptor returns default gRPC server interceptor
func GetDefaultServerInterceptor() []grpc.ServerOption {
options := []grpc.ServerOption{}
options = append(options, grpc.ChainUnaryInterceptor(openmetrics.UnaryServerInterceptor(metrics.DefaultGRPCServerMetrics),
grpcrecovery.UnaryServerInterceptor(grpcrecovery.WithRecoveryHandler(gRPCPanicRecoveryHandler))))
options = append(options, grpc.ChainStreamInterceptor(openmetrics.StreamServerInterceptor(metrics.DefaultGRPCServerMetrics)))
return options
}

// GetDefaultClientInterceptor returns default gRPC client interceptor
func GetDefaultClientInterceptor() []grpc.DialOption {
options := []grpc.DialOption{}
options = append(options, grpc.WithChainUnaryInterceptor(openmetrics.UnaryClientInterceptor(metrics.DefaultGRPCClientMetrics)))
options = append(options, grpc.WithChainStreamInterceptor(openmetrics.StreamClientInterceptor(metrics.DefaultGRPCClientMetrics)))
return options
}
12 changes: 6 additions & 6 deletions pkg/p2p/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import (
"strings"
"time"

signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
ggio "github.com/gogo/protobuf/io"
"github.com/gogo/protobuf/proto"
ds "github.com/ipfs/go-datastore"
leveldb "github.com/ipfs/go-ds-leveldb"
libp2p "github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/peerstore"
Expand All @@ -19,7 +19,7 @@ import (

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/p2p/types"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
)

// Node defines the p2p protocol node, encapsulates the go-lib.p2p
Expand Down Expand Up @@ -102,7 +102,7 @@ func (n *Node) Name() string {

// Start runs background task that trigger broadcast ping request
func (n *Node) Start(ctx context.Context) error {
go n.eventloop()
go n.eventLoop()
return nil
}

Expand Down Expand Up @@ -168,8 +168,8 @@ func (n *Node) GetApproval(object *storagetypes.ObjectInfo, expectedAccept int,
}
}

// eventloop run the background task
func (n *Node) eventloop() {
// eventLoop run the background task
func (n *Node) eventLoop() {
ticker := time.NewTicker(time.Duration(n.config.PingPeriod) * time.Second)
for {
select {
Expand Down
46 changes: 28 additions & 18 deletions service/challenge/challenge.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@ import (
"context"
"net"

"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"

"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

var _ lifecycle.Service = &Challenge{}
Expand All @@ -23,6 +26,7 @@ type Challenge struct {
config *ChallengeConfig
spDB sqldb.SPDB
pieceStore *psclient.StoreClient
grpcServer *grpc.Server
}

// NewChallengeService returns an instance of Challenge that implementation of
Expand Down Expand Up @@ -56,28 +60,34 @@ func (challenge *Challenge) Name() string {
// Start the challenge gRPC service
func (challenge *Challenge) Start(ctx context.Context) error {
errCh := make(chan error)

go func(errCh chan error) {
lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}
grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
types.RegisterChallengeServiceServer(grpcServer, challenge)
reflection.Register(grpcServer)
if err = grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}(errCh)

go challenge.serve(errCh)
err := <-errCh
return err
}

// Stop the challenge gRPC service and recycle the resources
func (challenge *Challenge) Stop(ctx context.Context) error {
challenge.grpcServer.GracefulStop()
return nil
}

func (challenge *Challenge) serve(errCh chan error) {
lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}

options := utilgrpc.GetDefaultServerOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultServerInterceptor()...)
}
challenge.grpcServer = grpc.NewServer(options...)
types.RegisterChallengeServiceServer(challenge.grpcServer, challenge)
reflection.Register(challenge.grpcServer)
if err = challenge.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}
16 changes: 9 additions & 7 deletions service/challenge/client/challenge_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package client
import (
"context"

"github.com/bnb-chain/greenfield-storage-provider/model"
merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

// ChallengeClient is a challenge gRPC service client wrapper
Expand All @@ -21,10 +22,11 @@ type ChallengeClient struct {

// NewChallengeClient return a ChallengeClient instance
func NewChallengeClient(address string) (*ChallengeClient, error) {
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
options := utilgrpc.GetDefaultClientOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultClientInterceptor()...)
}
conn, err := grpc.DialContext(context.Background(), address, options...)
if err != nil {
log.Errorw("failed to dial challenge", "error", err)
return nil, err
Expand Down
16 changes: 9 additions & 7 deletions service/downloader/client/downloader_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package client
import (
"context"

"github.com/bnb-chain/greenfield-storage-provider/model"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

// DownloaderClient is a downloader gRPC service client wrapper
Expand All @@ -21,10 +22,11 @@ type DownloaderClient struct {

// NewDownloaderClient returns a DownloaderClient instance
func NewDownloaderClient(address string) (*DownloaderClient, error) {
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
options := utilgrpc.GetDefaultClientOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultClientInterceptor()...)
}
conn, err := grpc.DialContext(context.Background(), address, options...)
if err != nil {
log.Errorw("failed to dial downloader", "error", err)
return nil, err
Expand Down
48 changes: 29 additions & 19 deletions service/downloader/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@ import (
"context"
"net"

"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"

"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

var _ lifecycle.Service = &Downloader{}
Expand All @@ -23,6 +26,7 @@ type Downloader struct {
config *DownloaderConfig
spDB sqldb.SPDB
pieceStore *psclient.StoreClient
grpcServer *grpc.Server
}

// NewDownloaderService returns an instance of Downloader that implementation of
Expand Down Expand Up @@ -56,28 +60,34 @@ func (downloader *Downloader) Name() string {
// Start the downloader gRPC service
func (downloader *Downloader) Start(ctx context.Context) error {
errCh := make(chan error)

go func(errCh chan error) {
lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}
grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
types.RegisterDownloaderServiceServer(grpcServer, downloader)
reflection.Register(grpcServer)
if err = grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}(errCh)

go downloader.serve(errCh)
err := <-errCh
return err
}

// Stop the downloader gRPC service and recycle the resources
func (downloader *Downloader) Stop(ctx context.Context) error {
downloader.grpcServer.GracefulStop()
return nil
}

func (downloader *Downloader) serve(errCh chan error) {
lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}

options := utilgrpc.GetDefaultServerOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultServerInterceptor()...)
}
downloader.grpcServer = grpc.NewServer(options...)
types.RegisterDownloaderServiceServer(downloader.grpcServer, downloader)
reflection.Register(downloader.grpcServer)
if err = downloader.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}
6 changes: 6 additions & 0 deletions service/p2p/client/p2p_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package client
import (
"context"

"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
Expand All @@ -22,6 +24,10 @@ type P2PClient struct {

// NewP2PClient return a P2PClient instance
func NewP2PClient(address string) (*P2PClient, error) {
options := []grpc.DialOption{}
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultClientInterceptor()...)
}
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
Expand Down
15 changes: 10 additions & 5 deletions service/p2p/p2p.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"time"

"github.com/bnb-chain/greenfield-common/go/hash"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mdgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"

Expand Down Expand Up @@ -92,11 +94,14 @@ func (p *P2PServer) serve(errCh chan error) {
return
}

grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize))
p2ptypes.RegisterP2PServiceServer(grpcServer, p)
p.grpcServer = grpcServer
reflection.Register(grpcServer)
if err := grpcServer.Serve(lis); err != nil {
options := []grpc.ServerOption{}
if metrics.GetMetrics().Enabled() {
options = append(options, mdgrpc.GetDefaultServerInterceptor()...)
}
p.grpcServer = grpc.NewServer(options...)
p2ptypes.RegisterP2PServiceServer(p.grpcServer, p)
reflection.Register(p.grpcServer)
if err := p.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to start grpc server", "err", err)
return
}
Expand Down
Loading

0 comments on commit 0f00a69

Please sign in to comment.