Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: sp service adds metrics #239

Merged
merged 2 commits into from
Mar 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions config/config_template.toml
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,14 @@ Modules = ["epoch", "bucket", "object", "payment"]
Dsn = "localhost:3308"

[P2PCfg]
ListenAddress = "127.0.0.1:9933"
ListenAddress = "127.0.0.1:9833"
P2PPrivateKey = ""
Bootstrap = []
PingPeriod = 2

[MetricsCfg]
Enabled = false
HTTPAddress = "localhost:9833"
HTTPAddress = "localhost:24036"

[LogCfg]
Level = "debug"
Expand Down
39 changes: 39 additions & 0 deletions pkg/middleware/grpc/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package grpc

import (
"runtime/debug"

openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2"
grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
)

var (
gRPCPanicRecoveryHandler = func(p interface{}) (err error) {
metrics.PanicsTotal.WithLabelValues().Inc()
log.Errorw("recovered from panic", "panic", p, "stack", debug.Stack())
return status.Errorf(codes.Internal, "%s", p)
}
)

// GetDefaultServerInterceptor returns default gRPC server interceptor
func GetDefaultServerInterceptor() []grpc.ServerOption {
options := []grpc.ServerOption{}
options = append(options, grpc.ChainUnaryInterceptor(openmetrics.UnaryServerInterceptor(metrics.DefaultGRPCServerMetrics),
grpcrecovery.UnaryServerInterceptor(grpcrecovery.WithRecoveryHandler(gRPCPanicRecoveryHandler))))
options = append(options, grpc.ChainStreamInterceptor(openmetrics.StreamServerInterceptor(metrics.DefaultGRPCServerMetrics)))
return options
}

// GetDefaultClientInterceptor returns default gRPC client interceptor
func GetDefaultClientInterceptor() []grpc.DialOption {
options := []grpc.DialOption{}
options = append(options, grpc.WithChainUnaryInterceptor(openmetrics.UnaryClientInterceptor(metrics.DefaultGRPCClientMetrics)))
options = append(options, grpc.WithChainStreamInterceptor(openmetrics.StreamClientInterceptor(metrics.DefaultGRPCClientMetrics)))
return options
}
12 changes: 6 additions & 6 deletions pkg/p2p/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import (
"strings"
"time"

signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
ggio "github.com/gogo/protobuf/io"
"github.com/gogo/protobuf/proto"
ds "github.com/ipfs/go-datastore"
leveldb "github.com/ipfs/go-ds-leveldb"
libp2p "github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/peerstore"
Expand All @@ -19,7 +19,7 @@ import (

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/p2p/types"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
)

// Node defines the p2p protocol node, encapsulates the go-lib.p2p
Expand Down Expand Up @@ -102,7 +102,7 @@ func (n *Node) Name() string {

// Start runs background task that trigger broadcast ping request
func (n *Node) Start(ctx context.Context) error {
go n.eventloop()
go n.eventLoop()
return nil
}

Expand Down Expand Up @@ -168,8 +168,8 @@ func (n *Node) GetApproval(object *storagetypes.ObjectInfo, expectedAccept int,
}
}

// eventloop run the background task
func (n *Node) eventloop() {
// eventLoop run the background task
func (n *Node) eventLoop() {
ticker := time.NewTicker(time.Duration(n.config.PingPeriod) * time.Second)
for {
select {
Expand Down
48 changes: 29 additions & 19 deletions service/challenge/challenge.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,19 @@ import (
"context"
"net"

"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"

"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

var _ lifecycle.Service = &Challenge{}
Expand All @@ -25,6 +28,7 @@ type Challenge struct {
spDB sqldb.SPDB
rcScope rcmgr.ResourceScope
pieceStore *psclient.StoreClient
grpcServer *grpc.Server
}

// NewChallengeService returns an instance of Challenge that implementation of
Expand Down Expand Up @@ -62,29 +66,35 @@ func (challenge *Challenge) Name() string {
// Start the challenge gRPC service
func (challenge *Challenge) Start(ctx context.Context) error {
errCh := make(chan error)

go func(errCh chan error) {
lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}
grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
types.RegisterChallengeServiceServer(grpcServer, challenge)
reflection.Register(grpcServer)
if err = grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}(errCh)

go challenge.serve(errCh)
err := <-errCh
return err
}

// Stop the challenge gRPC service and recycle the resources
func (challenge *Challenge) Stop(ctx context.Context) error {
challenge.grpcServer.GracefulStop()
challenge.rcScope.Release()
return nil
}

func (challenge *Challenge) serve(errCh chan error) {
lis, err := net.Listen("tcp", challenge.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}

options := utilgrpc.GetDefaultServerOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
}
challenge.grpcServer = grpc.NewServer(options...)
types.RegisterChallengeServiceServer(challenge.grpcServer, challenge)
reflection.Register(challenge.grpcServer)
if err = challenge.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}
16 changes: 9 additions & 7 deletions service/challenge/client/challenge_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package client
import (
"context"

"github.com/bnb-chain/greenfield-storage-provider/model"
merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

merrors "github.com/bnb-chain/greenfield-storage-provider/model/errors"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/challenge/types"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

// ChallengeClient is a challenge gRPC service client wrapper
Expand All @@ -21,10 +22,11 @@ type ChallengeClient struct {

// NewChallengeClient return a ChallengeClient instance
func NewChallengeClient(address string) (*ChallengeClient, error) {
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
options := utilgrpc.GetDefaultClientOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
}
conn, err := grpc.DialContext(context.Background(), address, options...)
if err != nil {
log.Errorw("failed to dial challenge", "error", err)
return nil, err
Expand Down
16 changes: 9 additions & 7 deletions service/downloader/client/downloader_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package client
import (
"context"

"github.com/bnb-chain/greenfield-storage-provider/model"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
storagetypes "github.com/bnb-chain/greenfield/x/storage/types"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

// DownloaderClient is a downloader gRPC service client wrapper
Expand All @@ -21,10 +22,11 @@ type DownloaderClient struct {

// NewDownloaderClient returns a DownloaderClient instance
func NewDownloaderClient(address string) (*DownloaderClient, error) {
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
options := utilgrpc.GetDefaultClientOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
}
conn, err := grpc.DialContext(context.Background(), address, options...)
if err != nil {
log.Errorw("failed to dial downloader", "error", err)
return nil, err
Expand Down
50 changes: 30 additions & 20 deletions service/downloader/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,19 @@ import (
"context"
"net"

"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"

"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/pkg/rcmgr"
"github.com/bnb-chain/greenfield-storage-provider/service/downloader/types"
psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client"
"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
utilgrpc "github.com/bnb-chain/greenfield-storage-provider/util/grpc"
)

var _ lifecycle.Service = &Downloader{}
Expand All @@ -24,6 +27,7 @@ type Downloader struct {
config *DownloaderConfig
spDB sqldb.SPDB
pieceStore *psclient.StoreClient
grpcServer *grpc.Server
rcScope rcmgr.ResourceScope
}

Expand Down Expand Up @@ -60,29 +64,35 @@ func (downloader *Downloader) Name() string {
// Start the downloader gRPC service
func (downloader *Downloader) Start(ctx context.Context) error {
errCh := make(chan error)

go func(errCh chan error) {
lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}
grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize))
types.RegisterDownloaderServiceServer(grpcServer, downloader)
reflection.Register(grpcServer)
if err = grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}(errCh)

go downloader.serve(errCh)
err := <-errCh
return err
}

// Stop the downloader gRPC service and recycle the resources
func (downloader *Downloader) Stop(ctx context.Context) error {
downloader.grpcServer.GracefulStop()
downloader.rcScope.Release()
return nil
}

func (downloader *Downloader) serve(errCh chan error) {
lis, err := net.Listen("tcp", downloader.config.GRPCAddress)
errCh <- err
if err != nil {
log.Errorw("failed to listen", "error", err)
return
}

options := utilgrpc.GetDefaultServerOptions()
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
}
downloader.grpcServer = grpc.NewServer(options...)
types.RegisterDownloaderServiceServer(downloader.grpcServer, downloader)
reflection.Register(downloader.grpcServer)
if err = downloader.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to serve", "error", err)
return
}
}
13 changes: 8 additions & 5 deletions service/p2p/client/p2p_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
p2ptypes "github.com/bnb-chain/greenfield-storage-provider/pkg/p2p/types"
"github.com/bnb-chain/greenfield-storage-provider/service/p2p/types"
)
Expand All @@ -22,10 +23,12 @@ type P2PClient struct {

// NewP2PClient return a P2PClient instance
func NewP2PClient(address string) (*P2PClient, error) {
conn, err := grpc.DialContext(context.Background(), address,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize)))
options := []grpc.DialOption{}
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultClientInterceptor()...)
}
options = append(options, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.DialContext(context.Background(), address, options...)
if err != nil {
log.Errorw("failed to dial p2p server", "error", err)
return nil, err
Expand Down
15 changes: 10 additions & 5 deletions service/p2p/p2p.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/bnb-chain/greenfield-storage-provider/model"
"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
mwgrpc "github.com/bnb-chain/greenfield-storage-provider/pkg/middleware/grpc"
"github.com/bnb-chain/greenfield-storage-provider/pkg/p2p"
p2ptypes "github.com/bnb-chain/greenfield-storage-provider/service/p2p/types"
signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client"
Expand Down Expand Up @@ -92,11 +94,14 @@ func (p *P2PServer) serve(errCh chan error) {
return
}

grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize))
p2ptypes.RegisterP2PServiceServer(grpcServer, p)
p.grpcServer = grpcServer
reflection.Register(grpcServer)
if err := grpcServer.Serve(lis); err != nil {
options := []grpc.ServerOption{}
if metrics.GetMetrics().Enabled() {
options = append(options, mwgrpc.GetDefaultServerInterceptor()...)
}
p.grpcServer = grpc.NewServer(options...)
p2ptypes.RegisterP2PServiceServer(p.grpcServer, p)
reflection.Register(p.grpcServer)
if err := p.grpcServer.Serve(lis); err != nil {
log.Errorw("failed to start grpc server", "err", err)
return
}
Expand Down
Loading