From 6fcfe19bcb0bd0e5d1bf8b3ac9290b168a07fc4f Mon Sep 17 00:00:00 2001 From: VM <112189277+sysvm@users.noreply.github.com> Date: Thu, 23 Mar 2023 21:08:40 +0800 Subject: [PATCH] feat: sp services add metrics (#211) * feat: sp service adds metrics * fix: add metrics config * refactor: refactor metrics service * fix: update go.mod * fix: fix config toml template * chore: coding adjustments * fix: golang ci lint error --------- Co-authored-by: DylanYong Co-authored-by: joeylichang --- cmd/conf/conf.go | 2 +- cmd/storage_provider/init.go | 21 ++- cmd/storage_provider/main.go | 31 ++++- cmd/utils/db.go | 4 +- cmd/utils/flags.go | 107 ++++++++++----- config/config.go | 9 ++ config/config_template.toml | 4 + config/subconfig.go | 2 +- deployment/localup/localup.sh | 7 +- go.mod | 13 +- go.sum | 23 +++- model/const.go | 4 + model/errors/rpc_error.go | 4 +- pkg/lifecycle/lifecycle.go | 23 +++- pkg/metrics/http/http.go | 69 ++++++++++ pkg/metrics/http/options.go | 84 ++++++++++++ pkg/metrics/http/server_metrics.go | 148 +++++++++++++++++++++ pkg/metrics/http/server_options.go | 23 ++++ pkg/metrics/metric_items.go | 23 ++++ pkg/metrics/metrics.go | 137 +++++++++++++++++++ pkg/metrics/metrics_config.go | 7 + service/gateway/gateway.go | 12 +- service/uploader/client/uploader_client.go | 21 ++- service/uploader/uploader.go | 31 ++++- 24 files changed, 729 insertions(+), 80 deletions(-) create mode 100644 pkg/metrics/http/http.go create mode 100644 pkg/metrics/http/options.go create mode 100644 pkg/metrics/http/server_metrics.go create mode 100644 pkg/metrics/http/server_options.go create mode 100644 pkg/metrics/metric_items.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/metrics_config.go diff --git a/cmd/conf/conf.go b/cmd/conf/conf.go index 9514dd20c..3b4af984e 100644 --- a/cmd/conf/conf.go +++ b/cmd/conf/conf.go @@ -31,7 +31,7 @@ var ConfigUploadCmd = &cli.Command{ utils.DBUserFlag, utils.DBPasswordFlag, utils.DBAddressFlag, - utils.DBDataBaseFlag, + utils.DBDatabaseFlag, }, Category: "CONFIG COMMANDS", Description: ` diff --git a/cmd/storage_provider/init.go b/cmd/storage_provider/init.go index 665234d02..d5f2ec2d0 100644 --- a/cmd/storage_provider/init.go +++ b/cmd/storage_provider/init.go @@ -10,6 +10,7 @@ import ( "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics" "github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer" "github.com/bnb-chain/greenfield-storage-provider/service/challenge" "github.com/bnb-chain/greenfield-storage-provider/service/downloader" @@ -23,7 +24,7 @@ import ( "github.com/bnb-chain/greenfield-storage-provider/service/uploader" ) -// initLog init global log level and log path. +// initLog initializes global log level and log path. func initLog(ctx *cli.Context, cfg *config.StorageProviderConfig) error { if cfg.LogCfg == nil { cfg.LogCfg = config.DefaultLogConfig @@ -45,6 +46,24 @@ func initLog(ctx *cli.Context, cfg *config.StorageProviderConfig) error { return nil } +// initMetrics initializes global metrics. +func initMetrics(ctx *cli.Context, cfg *config.StorageProviderConfig) error { + if cfg == nil { + cfg.MetricsCfg = config.DefaultMetricsConfig + } + if ctx.IsSet(utils.MetricsEnabledFlag.Name) { + cfg.MetricsCfg.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) + } + if ctx.IsSet(utils.MetricsHTTPFlag.Name) { + cfg.MetricsCfg.HTTPAddress = ctx.String(utils.MetricsHTTPFlag.Name) + } + if cfg.MetricsCfg.Enabled { + slc := lifecycle.NewServiceLifecycle() + slc.RegisterServices(metrics.NewMetrics(cfg.MetricsCfg)) + } + return nil +} + // initService init service instance by name and config. func initService(serviceName string, cfg *config.StorageProviderConfig) (server lifecycle.Service, err error) { switch serviceName { diff --git a/cmd/storage_provider/main.go b/cmd/storage_provider/main.go index 4c4a8685b..8d8bec2ce 100644 --- a/cmd/storage_provider/main.go +++ b/cmd/storage_provider/main.go @@ -24,20 +24,31 @@ var ( var app *cli.App +// flags that configure the storage provider var ( - // flags that configure the storage provider - spFlags = []cli.Flag{ + configFlags = []cli.Flag{ utils.ConfigFileFlag, utils.ConfigRemoteFlag, utils.ServerFlag, + } + + dbFlags = []cli.Flag{ utils.DBUserFlag, utils.DBPasswordFlag, utils.DBAddressFlag, - utils.DBDataBaseFlag, + utils.DBDatabaseFlag, + } + + logFlags = []cli.Flag{ utils.LogLevelFlag, utils.LogPathFlag, utils.LogStdOutputFlag, } + + metricsFlags = []cli.Flag{ + utils.MetricsEnabledFlag, + utils.MetricsHTTPFlag, + } ) func init() { @@ -46,7 +57,12 @@ func init() { app.Usage = appUsage app.Action = storageProvider app.HideVersion = true - app.Flags = append(app.Flags, spFlags...) + app.Flags = utils.MergeFlags( + configFlags, + dbFlags, + logFlags, + metricsFlags, + ) app.Commands = []*cli.Command{ // config category commands conf.ConfigDumpCmd, @@ -85,6 +101,7 @@ func makeConfig(ctx *cli.Context) (*config.StorageProviderConfig, error) { } else if ctx.IsSet(utils.ConfigFileFlag.Name) { cfg = config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) } + // override the services to be started by flag if ctx.IsSet(utils.ServerFlag.Name) { services := util.SplitByComma(ctx.String(utils.ServerFlag.Name)) @@ -94,6 +111,10 @@ func makeConfig(ctx *cli.Context) (*config.StorageProviderConfig, error) { if err := initLog(ctx, cfg); err != nil { return nil, err } + // init metrics + if err := initMetrics(ctx, cfg); err != nil { + return nil, err + } return cfg, nil } @@ -113,7 +134,7 @@ func storageProvider(ctx *cli.Context) error { log.Errorw("failed to init service", "service", serviceName, "error", err) os.Exit(1) } - log.Debugw("success to init service ", "service", serviceName) + log.Debugw("succeed to init service ", "service", serviceName) // register service to lifecycle. slc.RegisterServices(service) } diff --git a/cmd/utils/db.go b/cmd/utils/db.go index 5ae214bcc..87b37ad0b 100644 --- a/cmd/utils/db.go +++ b/cmd/utils/db.go @@ -18,8 +18,8 @@ func MakeSPDB(ctx *cli.Context, spDBCfg *storeconfig.SQLDBConfig) (*sqldb.SpDBIm if ctx.IsSet(ctx.String(DBAddressFlag.Name)) { spDBCfg.Address = ctx.String(DBAddressFlag.Name) } - if ctx.IsSet(ctx.String(DBDataBaseFlag.Name)) { - spDBCfg.Database = ctx.String(DBDataBaseFlag.Name) + if ctx.IsSet(ctx.String(DBDatabaseFlag.Name)) { + spDBCfg.Database = ctx.String(DBDatabaseFlag.Name) } return sqldb.NewSpDB(spDBCfg) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8ed140827..af3ea8964 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1,62 +1,107 @@ package utils import ( + "github.com/bnb-chain/greenfield-storage-provider/config" "github.com/urfave/cli/v2" "github.com/bnb-chain/greenfield-storage-provider/model" ) +const ( + ConfigCategory = "SP CONFIG" + LoggingCategory = "LOGGING AND DEBUGGING" + MetricsCategory = "METRICS AND STATS" + DatabaseCategory = "DATABASE" +) + var ( ConfigFileFlag = &cli.StringFlag{ - Name: "config", - Aliases: []string{"c"}, - Usage: "Config file path for uploading to db", - Value: "./config.toml", + Name: "config", + Category: ConfigCategory, + Aliases: []string{"c"}, + Usage: "Config file path for uploading to db", + Value: "./config.toml", } ConfigRemoteFlag = &cli.BoolFlag{ - Name: "config.remote", + Name: "config.remote", + Category: ConfigCategory, Usage: "Flag load config from remote db,if 'config.remote' be set, the db.user, " + "db.password and db.address flags are needed, otherwise use the default value", } ServerFlag = &cli.StringFlag{ - Name: "server", - Aliases: []string{"service", "s"}, - Usage: "Services to be started list, eg -server gateway,uploader,receiver... ", + Name: "server", + Category: ConfigCategory, + Aliases: []string{"service", "svc"}, + Usage: "Services to be started list, e.g. -server gateway, uploader, receiver...", } + + // database flags DBUserFlag = &cli.StringFlag{ - Name: "db.user", - Usage: "DB user name", - EnvVars: []string{model.SpDBUser}, + Name: "db.user", + Category: DatabaseCategory, + Usage: "DB user name", + EnvVars: []string{model.SpDBUser}, } DBPasswordFlag = &cli.StringFlag{ - Name: "db.password", - Usage: "DB user password", - EnvVars: []string{model.SpDBPasswd}, + Name: "db.password", + Category: DatabaseCategory, + Usage: "DB user password", + EnvVars: []string{model.SpDBPasswd}, } DBAddressFlag = &cli.StringFlag{ - Name: "db.address", - Usage: "DB listen address", - EnvVars: []string{model.SpDBAddress}, - Value: "localhost:3306", + Name: "db.address", + Category: DatabaseCategory, + Usage: "DB listen address", + EnvVars: []string{model.SpDBAddress}, + Value: config.DefaultSQLDBConfig.Address, } - DBDataBaseFlag = &cli.StringFlag{ - Name: "db.database", - Usage: "DB database name", - EnvVars: []string{model.SpDBDataBase}, - Value: "localhost:3306", + DBDatabaseFlag = &cli.StringFlag{ + Name: "db.database", + Category: DatabaseCategory, + Usage: "DB database name", + EnvVars: []string{model.SpDBDataBase}, + Value: config.DefaultSQLDBConfig.Database, } + + // log flags LogLevelFlag = &cli.StringFlag{ - Name: "log.level", - Usage: "log level", - Value: "info", + Name: "log.level", + Category: LoggingCategory, + Usage: "log level", + Value: "info", } LogPathFlag = &cli.StringFlag{ - Name: "log.path", - Usage: "log output file path", - Value: "./gnfd-sp.log", + Name: "log.path", + Category: LoggingCategory, + Usage: "log output file path", + Value: config.DefaultLogConfig.Path, } LogStdOutputFlag = &cli.BoolFlag{ - Name: "log.std", - Usage: "log output standard io", + Name: "log.std", + Category: LoggingCategory, + Usage: "log output standard io", + } + + // Metrics flags + MetricsEnabledFlag = &cli.BoolFlag{ + Name: "metrics", + Category: MetricsCategory, + Usage: "Enable metrics collection and reporting", + Value: config.DefaultMetricsConfig.Enabled, + } + MetricsHTTPFlag = &cli.StringFlag{ + Name: "metrics.addr", + Category: MetricsCategory, + Usage: "Enable stand-alone metrics HTTP server listening address", + Value: config.DefaultMetricsConfig.HTTPAddress, } ) + +// MergeFlags merges the given flag slices. +func MergeFlags(groups ...[]cli.Flag) []cli.Flag { + var ret []cli.Flag + for _, group := range groups { + ret = append(ret, group...) + } + return ret +} diff --git a/config/config.go b/config/config.go index 224a953ec..0bc477c88 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ import ( "github.com/bnb-chain/greenfield-storage-provider/model" gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" + "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics" "github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer" "github.com/bnb-chain/greenfield-storage-provider/service/signer" "github.com/bnb-chain/greenfield-storage-provider/store/config" @@ -32,6 +33,7 @@ type StorageProviderConfig struct { BlockSyncerCfg *blocksyncer.Config P2PCfg *p2p.NodeConfig LogCfg *LogConfig + MetricsCfg *metrics.MetricsConfig } // JSONMarshal marshal the StorageProviderConfig to json format @@ -88,6 +90,7 @@ var DefaultStorageProviderConfig = &StorageProviderConfig{ BlockSyncerCfg: DefaultBlockSyncerConfig, P2PCfg: DefaultP2PConfig, LogCfg: DefaultLogConfig, + MetricsCfg: DefaultMetricsConfig, } // DefaultSQLDBConfig defines the default configuration of SQL DB @@ -123,6 +126,12 @@ var DefaultBlockSyncerConfig = &blocksyncer.Config{ Dsn: "localhost:3308", } +// DefaultMetricsConfig defines the default config of Metrics service +var DefaultMetricsConfig = &metrics.MetricsConfig{ + Enabled: false, + HTTPAddress: model.MetricsHTTPAddress, +} + type LogConfig struct { Level string Path string diff --git a/config/config_template.toml b/config/config_template.toml index ee8851745..ddd808360 100644 --- a/config/config_template.toml +++ b/config/config_template.toml @@ -65,6 +65,10 @@ P2PPrivateKey = "" Bootstrap = [] PingPeriod = 2 +[MetricsCfg] +Enabled = false +HTTPAddress = "localhost:9833" + [LogCfg] Level = "debug" Path = "./gnfd-sp.log" diff --git a/config/subconfig.go b/config/subconfig.go index d77f95b3c..e6228cbf3 100644 --- a/config/subconfig.go +++ b/config/subconfig.go @@ -190,7 +190,7 @@ func (cfg *StorageProviderConfig) MakeMetadataServiceConfig() (*metadata.Metadat if _, ok := cfg.ListenAddress[model.MetadataService]; ok { mCfg.GRPCAddress = cfg.ListenAddress[model.MetadataService] } else { - return nil, fmt.Errorf("missing meta data gRPC address configuration for meta data service") + return nil, fmt.Errorf("missing metadata gRPC address configuration for meta data service") } return mCfg, nil } diff --git a/deployment/localup/localup.sh b/deployment/localup/localup.sh index 28875a3d5..5d959dfec 100644 --- a/deployment/localup/localup.sh +++ b/deployment/localup/localup.sh @@ -57,9 +57,9 @@ generate_env() { done } -############################################################### -# make sp config.toml real according env.info/db.info/sp.info # -############################################################### +################################################################## +# make sp config.toml real according to env.info/db.info/sp.info # +################################################################## make_config() { index=0 for sp_dir in ${workspace}/${SP_DEPLOY_DIR}/* ; do @@ -83,6 +83,7 @@ make_config() { sed -i -e "s/9733/$(($cur_port+733))/g" config.toml sed -i -e "s/9833/$(($cur_port+833))/g" config.toml sed -i -e "s/9933/$(($cur_port+933))/g" config.toml + sed -i -e "s/24036/$(($cur_port+4036))/g" config.toml sed -i -e "s/SpOperatorAddress = \".*\"/SpOperatorAddress = \"${OPERATOR_ADDRESS}\"/g" config.toml sed -i -e "s/OperatorPrivateKey = \".*\"/OperatorPrivateKey = \"${OPERATOR_PRIVATE_KEY}\"/g" config.toml sed -i -e "s/FundingPrivateKey = \".*\"/FundingPrivateKey = \"${FUNDING_PRIVATE_KEY}\"/g" config.toml diff --git a/go.mod b/go.mod index 0af3a8c71..65aeae6d6 100644 --- a/go.mod +++ b/go.mod @@ -20,10 +20,8 @@ require ( github.com/bnb-chain/greenfield-go-sdk v0.0.0-20230314083410-f0d6dbbec179 github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6 github.com/cloudflare/cfssl v1.6.3 - github.com/cosmos/cosmos-proto v1.0.0-beta.1 github.com/cosmos/cosmos-sdk v0.46.7 github.com/cosmos/gogoproto v1.4.6 - github.com/crx666/protobuf-gogofast v1.2.7 github.com/ethereum/go-ethereum v1.10.19 github.com/forbole/juno/v4 v4.0.0-00010101000000-000000000000 github.com/gin-gonic/gin v1.8.2 @@ -31,6 +29,7 @@ require ( github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2 v2.0.0-rc.3 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.0 @@ -39,10 +38,12 @@ require ( github.com/multiformats/go-multiaddr v0.8.0 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/pkg/sftp v1.13.5 + github.com/prometheus/client_golang v1.14.0 github.com/stretchr/testify v1.8.1 github.com/tendermint/tendermint v0.35.9 - github.com/urfave/cli/v2 v2.3.0 + github.com/urfave/cli/v2 v2.25.0 github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8 + go.opentelemetry.io/otel/trace v1.11.0 go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 @@ -58,7 +59,7 @@ require ( ) require ( - cosmossdk.io/errors v1.0.0-beta.7 // indirect + cosmossdk.io/errors v1.0.0-beta.7 filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect @@ -130,6 +131,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect @@ -206,7 +208,6 @@ require ( github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect @@ -250,9 +251,11 @@ require ( github.com/wealdtech/go-eth2-types/v2 v2.5.2 // indirect github.com/wealdtech/go-eth2-util v1.6.3 // indirect github.com/willf/bitset v1.1.11 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.14.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect + go.opentelemetry.io/otel v1.11.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect diff --git a/go.sum b/go.sum index c30bde79f..cf0c43af4 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,6 @@ github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crx666/protobuf-gogofast v1.2.7 h1:/rglPP6qoMz5A7HI0U3JfbsvIjlWueoLoc9pxqnv02g= -github.com/crx666/protobuf-gogofast v1.2.7/go.mod h1:wgjYcm8tA5wxXqg2iXkCsdK6kyWTVdQa/XFpLPP0k+w= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= @@ -737,6 +735,11 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2 v2.0.0-rc.3 h1:kKuOg7gEBO7otn5QpZ4FnlbZBz1p5EZ7sX6RDbE36Bc= +github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2 v2.0.0-rc.3/go.mod h1:LzR39RXGCAfCfK/NNauJ1qqENhVoMuBt40i1EYuX9bs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20210128111500-3ff779b52992/go.mod h1:fG+XdHpfvMfdBQ9GTIRX0nXjb+VHaMWV5o9OCgw6Qi8= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5YbLR0/n2+wBxUpgf3HgfKgTyLI= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -951,7 +954,6 @@ github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2vi github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca/go.mod h1:ph+C5vpnCcQvKBwJwKLTK3JLNGnBXYlG7m7JjoC/zYA= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= @@ -1752,8 +1754,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= +github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -1789,6 +1792,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/xtaci/kcp-go v5.4.20+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1834,6 +1839,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= +go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= +go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI= +go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -2240,7 +2249,6 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2415,8 +2423,10 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2459,6 +2469,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.30.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= @@ -2474,6 +2485,8 @@ google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20200723182653-9106c3fff523/go.mod h1:5j1uub0jRGhRiSghIlrThmBUgcgLXOVJQ/l1getT4uo= +google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/model/const.go b/model/const.go index 3249a4fbf..6d3ea2aaa 100644 --- a/model/const.go +++ b/model/const.go @@ -24,6 +24,8 @@ var ( BlockSyncerService = strings.ToLower("BlockSyncer") // ManagerService defines the name of manager service ManagerService = strings.ToLower("Manager") + // MetricsService defines the name of metrics service + MetricsService = strings.ToLower("Metrics") // P2PService defines the name of p2p service P2PService = strings.ToLower("p2p") ) @@ -60,6 +62,8 @@ const ( SignerGRPCAddress = "localhost:9633" // MetadataGRPCAddress default gRPC address of meta data service MetadataGRPCAddress = "localhost:9733" + // MetricsHTTPAddress default HTTP address of metrics service + MetricsHTTPAddress = "localhost:24036" // P2PGRPCAddress default gRPC address of p2p service P2PGRPCAddress = "localhost:9833" // P2PListenAddress default p2p protocol listen address of p2p node diff --git a/model/errors/rpc_error.go b/model/errors/rpc_error.go index 8536fcd2b..5cd2b320a 100644 --- a/model/errors/rpc_error.go +++ b/model/errors/rpc_error.go @@ -20,12 +20,12 @@ var ( ErrNoSuchBucket = errors.New("the specified bucket does not exist") // ErrInvalidBucketName defines invalid bucket name ErrInvalidBucketName = errors.New("invalid bucket name") + // ErrUnsupportedMethod defines unsupported method error + ErrUnsupportedMethod = errors.New("unsupported method") ) // piece store errors var ( - // ErrUnsupportedMethod defines unsupported method error - ErrUnsupportedMethod = errors.New("unsupported method") // ErrUnsupportedDelimiter defines invalid key with delimiter error ErrUnsupportedDelimiter = errors.New("unsupported delimiter") // ErrInvalidObjectKey defines invalid object key error diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 4e5256d0e..dc237faf2 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -5,6 +5,7 @@ import ( "errors" "os" "os/signal" + "sync" "time" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" @@ -32,14 +33,22 @@ type ServiceLifecycle struct { timeout time.Duration } -// NewServiceLifecycle returns an initialized service lifecycle +var ( + slc *ServiceLifecycle + once sync.Once +) + +// NewServiceLifecycle returns a singleton instance of ServiceLifecycle func NewServiceLifecycle() *ServiceLifecycle { - innerCtx, innerCancel := context.WithCancel(context.Background()) - return &ServiceLifecycle{ - innerCtx: innerCtx, - innerCancel: innerCancel, - timeout: time.Duration(StopTimeout) * time.Second, - } + once.Do(func() { + innerCtx, innerCancel := context.WithCancel(context.Background()) + slc = &ServiceLifecycle{ + innerCtx: innerCtx, + innerCancel: innerCancel, + timeout: time.Duration(StopTimeout) * time.Second, + } + }) + return slc } // RegisterServices register services of an application diff --git a/pkg/metrics/http/http.go b/pkg/metrics/http/http.go new file mode 100644 index 000000000..58fca7102 --- /dev/null +++ b/pkg/metrics/http/http.go @@ -0,0 +1,69 @@ +package http + +import ( + "net/http" + "strconv" +) + +// responseWriterDelegator implements http.ResponseWriter and extracts the statusCode. +type responseWriterDelegator struct { + w http.ResponseWriter + written bool + size int + statusCode int +} + +func (wd *responseWriterDelegator) Header() http.Header { + return wd.w.Header() +} + +func (wd *responseWriterDelegator) Write(bytes []byte) (int, error) { + if wd.statusCode == 0 { + wd.statusCode = http.StatusOK + } + n, err := wd.w.Write(bytes) + wd.size += n + return n, err +} + +func (wd *responseWriterDelegator) WriteHeader(statusCode int) { + wd.written = true + wd.statusCode = statusCode + wd.w.WriteHeader(statusCode) +} + +func (wd *responseWriterDelegator) StatusCode() int { + if !wd.written { + return http.StatusOK + } + return wd.statusCode +} + +func (wd *responseWriterDelegator) Status() string { + return strconv.Itoa(wd.StatusCode()) +} + +// computeApproximateRequestSize compute HTTP request size +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s += len(r.URL.String()) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} diff --git a/pkg/metrics/http/options.go b/pkg/metrics/http/options.go new file mode 100644 index 000000000..3e4d374e0 --- /dev/null +++ b/pkg/metrics/http/options.go @@ -0,0 +1,84 @@ +package http + +import "github.com/prometheus/client_golang/prometheus" + +// CounterOption lets you add options to Counter metrics using With* functions. +type CounterOption func(*prometheus.CounterOpts) + +type counterOptions []CounterOption + +func (co counterOptions) apply(o prometheus.CounterOpts) prometheus.CounterOpts { + for _, f := range co { + f(&o) + } + return o +} + +// WithCounterConstLabels allows you to add ConstLabels to Counter metrics. +func WithCounterConstLabels(labels prometheus.Labels) CounterOption { + return func(o *prometheus.CounterOpts) { + o.ConstLabels = labels + } +} + +// GaugeOption lets you add options to gauge metrics using With* functions. +type GaugeOption func(opts *prometheus.GaugeOpts) + +type gaugeOptions []GaugeOption + +func (g gaugeOptions) apply(o prometheus.GaugeOpts) prometheus.GaugeOpts { + for _, f := range g { + f(&o) + } + return o +} + +// WithGaugeConstLabels allows you to add ConstLabels to Gauge metrics. +func WithGaugeConstLabels(labels prometheus.Labels) CounterOption { + return func(o *prometheus.CounterOpts) { + o.ConstLabels = labels + } +} + +// SummaryOption lets you add options to gauge metrics using With* functions. +type SummaryOption func(opts *prometheus.SummaryOpts) + +type summaryOptions []SummaryOption + +func (s summaryOptions) apply(o prometheus.SummaryOpts) prometheus.SummaryOpts { + for _, f := range s { + f(&o) + } + return o +} + +// WithSummaryConstLabels allows you to add ConstLabels to Summary metrics. +func WithSummaryConstLabels(labels prometheus.Labels) CounterOption { + return func(o *prometheus.CounterOpts) { + o.ConstLabels = labels + } +} + +// HistogramOption lets you add options to Histogram metrics using With* functions. +type HistogramOption func(*prometheus.HistogramOpts) + +type histogramOptions []HistogramOption + +func (ho histogramOptions) apply(o prometheus.HistogramOpts) prometheus.HistogramOpts { + for _, f := range ho { + f(&o) + } + return o +} + +// WithHistogramBuckets allows you to specify custom bucket ranges for histograms if EnableHandlingTimeHistogram is on. +func WithHistogramBuckets(buckets []float64) HistogramOption { + return func(o *prometheus.HistogramOpts) { o.Buckets = buckets } +} + +// WithHistogramConstLabels allows you to add custom ConstLabels to histograms metrics. +func WithHistogramConstLabels(labels prometheus.Labels) HistogramOption { + return func(o *prometheus.HistogramOpts) { + o.ConstLabels = labels + } +} diff --git a/pkg/metrics/http/server_metrics.go b/pkg/metrics/http/server_metrics.go new file mode 100644 index 000000000..f59ebf7f0 --- /dev/null +++ b/pkg/metrics/http/server_metrics.go @@ -0,0 +1,148 @@ +package http + +import ( + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/trace" +) + +// ServerMetrics represents a collection of metrics to be registered on a +// Prometheus metrics registry for an HTTP server. +type ServerMetrics struct { + serverReqTotalCounter *prometheus.CounterVec + serverReqInflightGauge *prometheus.GaugeVec + serverReqSizeSummary *prometheus.SummaryVec + serverRespSizeSummary *prometheus.SummaryVec + serverReqDuration *prometheus.HistogramVec +} + +// NewServerMetrics returns an instance of ServerMetrics +func NewServerMetrics(opts ...ServerMetricsOption) *ServerMetrics { + var config serverMetricsConfig + config.apply(opts) + return &ServerMetrics{ + // host and path is not monitored because of virtual path + serverReqTotalCounter: prometheus.NewCounterVec( + config.counterOpts.apply(prometheus.CounterOpts{ + Name: "http_server_received_total_requests", + Help: "Tracks the total number of HTTP requests.", + }), []string{"handler_name", "method", "code"}), + serverReqInflightGauge: prometheus.NewGaugeVec( + config.gaugeOpts.apply(prometheus.GaugeOpts{ + Name: "http_server_inflight_requests", + Help: "Current number of HTTP requests the handler is responding to.", + }), []string{"handler_name", "method"}), + serverReqSizeSummary: prometheus.NewSummaryVec( + config.summaryOpts.apply(prometheus.SummaryOpts{ + Name: "http_request_size_bytes", + Help: "Tracks the size of HTTP requests.", + }), []string{"handler_name", "method", "code"}), + serverRespSizeSummary: prometheus.NewSummaryVec( + config.summaryOpts.apply(prometheus.SummaryOpts{ + Name: "http_response_size_bytes", + Help: "Tracks the size of HTTP responses.", + }), []string{"handler_name", "method", "code"}), + serverReqDuration: prometheus.NewHistogramVec( + config.histogramOpts.apply(prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "Tracks the latencies for HTTP requests.", + Buckets: prometheus.DefBuckets, + }), []string{"handler_name", "method", "code"}), + } +} + +// NewRegisteredServerMetrics returns a custom ServerMetrics object registered with the user's registry +// and registers some common metrics associated with every instance. +func NewRegisteredServerMetrics(registry prometheus.Registerer, opts ...ServerMetricsOption) *ServerMetrics { + customServerMetrics := NewServerMetrics(opts...) + customServerMetrics.MustRegister(registry) + return customServerMetrics +} + +// Register registers the metrics with the registry. +func (m *ServerMetrics) Register(registry prometheus.Registerer) error { + for _, collector := range m.toRegister() { + if err := registry.Register(collector); err != nil { + return err + } + } + return nil +} + +// MustRegister registers the metrics with the registry +// Panic if any error occurs much like DefaultRegisterer of Prometheus. +func (m *ServerMetrics) MustRegister(registry prometheus.Registerer) { + registry.MustRegister(m.toRegister()...) +} + +func (m *ServerMetrics) toRegister() []prometheus.Collector { + res := []prometheus.Collector{ + m.serverReqTotalCounter, + m.serverReqInflightGauge, + m.serverReqSizeSummary, + m.serverRespSizeSummary, + m.serverReqDuration, + } + return res +} + +// Describe sends the super-set of all possible descriptors of metrics collected by this Collector to the provided +// channel and returns once the last descriptor has been sent. +func (m *ServerMetrics) Describe(ch chan<- *prometheus.Desc) { + m.serverReqTotalCounter.Describe(ch) + m.serverReqInflightGauge.Describe(ch) + m.serverReqSizeSummary.Describe(ch) + m.serverRespSizeSummary.Describe(ch) + m.serverReqDuration.Describe(ch) +} + +// Collect is called by the Prometheus registry when collecting metrics. The implementation sends each +// collected metric via the provided channel and returns once the last metric has been sent. +func (m *ServerMetrics) Collect(ch chan<- prometheus.Metric) { + m.serverReqTotalCounter.Collect(ch) + m.serverReqInflightGauge.Collect(ch) + m.serverReqSizeSummary.Collect(ch) + m.serverRespSizeSummary.Collect(ch) + m.serverReqDuration.Collect(ch) +} + +// InstrumentationHandler initializes all metrics, with their appropriate null value, for all HTTP methods registered +// on an HTTP server. This is useful, to ensure that all metrics exist when collecting and querying. +func (m *ServerMetrics) InstrumentationHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + now := time.Now() + + wd := &responseWriterDelegator{w: w} + next.ServeHTTP(wd, r) + + method := r.Method + code := wd.Status() + handlerName := mux.CurrentRoute(r).GetName() + + m.serverReqTotalCounter.WithLabelValues(handlerName, method, code).Inc() + gauge := m.serverReqInflightGauge.WithLabelValues(handlerName, method) + gauge.Inc() + defer gauge.Dec() + m.serverReqSizeSummary.WithLabelValues(handlerName, method, code).Observe(float64(computeApproximateRequestSize(r))) + m.serverRespSizeSummary.WithLabelValues(handlerName, method, code).Observe(float64(wd.size)) + observer := m.serverReqDuration.WithLabelValues(handlerName, method, code) + observer.Observe(time.Since(now).Seconds()) + + var traceID string + span := trace.SpanFromContext(r.Context()) + if span != nil && span.SpanContext().IsSampled() { + traceID = span.SpanContext().TraceID().String() + } + if traceID != "" { + observer.(prometheus.ExemplarObserver).ObserveWithExemplar( + time.Since(now).Seconds(), + prometheus.Labels{ + "traceID": traceID, + }, + ) + } + }) +} diff --git a/pkg/metrics/http/server_options.go b/pkg/metrics/http/server_options.go new file mode 100644 index 000000000..2342b46df --- /dev/null +++ b/pkg/metrics/http/server_options.go @@ -0,0 +1,23 @@ +package http + +type serverMetricsConfig struct { + counterOpts counterOptions + gaugeOpts gaugeOptions + histogramOpts histogramOptions + summaryOpts summaryOptions +} + +type ServerMetricsOption func(*serverMetricsConfig) + +func (c *serverMetricsConfig) apply(opts []ServerMetricsOption) { + for _, o := range opts { + o(c) + } +} + +// WithServerCounterOptions adds options to counter +func WithServerCounterOptions(opts ...CounterOption) ServerMetricsOption { + return func(o *serverMetricsConfig) { + o.counterOpts = opts + } +} diff --git a/pkg/metrics/metric_items.go b/pkg/metrics/metric_items.go new file mode 100644 index 000000000..2542a822c --- /dev/null +++ b/pkg/metrics/metric_items.go @@ -0,0 +1,23 @@ +package metrics + +import ( + metricshttp "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics/http" + openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2" + "github.com/prometheus/client_golang/prometheus" +) + +// this file is used to write metric items in sp service +var ( + // DefaultGRPCServerMetrics create default gRPC server metrics + DefaultGRPCServerMetrics = openmetrics.NewServerMetrics(openmetrics.WithServerHandlingTimeHistogram()) + // DefaultGRPCClientMetrics create default gRPC client metrics + DefaultGRPCClientMetrics = openmetrics.NewClientMetrics(openmetrics.WithClientHandlingTimeHistogram(), + openmetrics.WithClientStreamSendHistogram(), openmetrics.WithClientStreamRecvHistogram()) + // DefaultHTTPServerMetrics create default HTTP server metrics + DefaultHTTPServerMetrics = metricshttp.NewServerMetrics() + // PanicsTotal record the number of rpc panics + PanicsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "grpc_req_panics_recovered_total", + Help: "Total number of gRPC requests recovered from internal panic.", + }, []string{"grpc_type", "grpc_service", "grpc_method"}) +) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..5a04c815f --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,137 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/bnb-chain/greenfield-storage-provider/model" + "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" + "github.com/bnb-chain/greenfield-storage-provider/pkg/log" +) + +var ( + mMonitor MetricsMonitor + once sync.Once +) + +// MetricsMonitor defines abstract method +type MetricsMonitor interface { + lifecycle.Service + Enabled() bool +} + +// Metrics is used to monitor sp services +type Metrics struct { + config *MetricsConfig + registry *prometheus.Registry + httpServer *http.Server +} + +// NewMetrics returns a singleton instance of Metrics. +// Note: enable metrics should call NewMetrics with MetricsConfig and MetricsConfig.Enabled is set true. +// GetMetrics will return the singleton instance of Metrics to use at anywhere. If NewMetrics is not called, +// then the metrics is disabled when calls GetMetrics. +func NewMetrics(cfg *MetricsConfig) MetricsMonitor { + return initMetrics(cfg) +} + +// GetMetrics gets an instance of MetricsMonitor +func GetMetrics() MetricsMonitor { + return initMetrics(nil) +} + +// initMetrics is used to init metrics according to MetricsConfig +func initMetrics(cfg *MetricsConfig) MetricsMonitor { + once.Do(func() { + if cfg == nil || !cfg.Enabled { + mMonitor = NilMetrics{} + } else { + mMonitor = &Metrics{ + config: cfg, + registry: prometheus.NewRegistry(), + } + } + }) + return mMonitor +} + +// Name describes metrics service name +func (m *Metrics) Name() string { + return model.MetricsService +} + +// Start HTTP server +func (m *Metrics) Start(ctx context.Context) error { + m.registerMetricItems() + go m.serve() + return nil +} + +// Stop HTTP server +func (m *Metrics) Stop(ctx context.Context) error { + var errs []error + if err := m.httpServer.Shutdown(ctx); err != nil { + errs = append(errs, err) + } + if errs != nil { + return fmt.Errorf("%v", errs) + } + return nil +} + +// Enabled returns whether starts prometheus metrics +func (m *Metrics) Enabled() bool { + if m.config != nil { + return m.config.Enabled + } else { + return false + } +} + +func (m *Metrics) registerMetricItems() { + m.registry.MustRegister(DefaultGRPCServerMetrics, DefaultGRPCClientMetrics, DefaultHTTPServerMetrics, + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), PanicsTotal) +} + +func (m *Metrics) serve() { + router := mux.NewRouter() + router.Path("/metrics").Handler(promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + m.httpServer = &http.Server{ + Addr: m.config.HTTPAddress, + Handler: router, + } + if err := m.httpServer.ListenAndServe(); err != nil { + log.Errorw("failed to listen and serve", "error", err) + return + } +} + +// NilMetrics is a no-op Metrics +type NilMetrics struct{} + +// Name is a no-op +func (NilMetrics) Name() string { + return "" +} + +// Start is a no-op +func (NilMetrics) Start(ctx context.Context) error { + return nil +} + +// Stop is a no-op +func (NilMetrics) Stop(ctx context.Context) error { + return nil +} + +// Enabled is a no-op +func (NilMetrics) Enabled() bool { + return false +} diff --git a/pkg/metrics/metrics_config.go b/pkg/metrics/metrics_config.go new file mode 100644 index 000000000..3189cd09b --- /dev/null +++ b/pkg/metrics/metrics_config.go @@ -0,0 +1,7 @@ +package metrics + +// MetricsConfig contains the configuration for the metric collection. +type MetricsConfig struct { + Enabled bool `toml:",omitempty"` + HTTPAddress string `toml:",omitempty"` +} diff --git a/service/gateway/gateway.go b/service/gateway/gateway.go index bcf2b327b..83c23de7f 100644 --- a/service/gateway/gateway.go +++ b/service/gateway/gateway.go @@ -13,6 +13,7 @@ import ( chainclient "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics" challengeclient "github.com/bnb-chain/greenfield-storage-provider/service/challenge/client" downloaderclient "github.com/bnb-chain/greenfield-storage-provider/service/downloader/client" metadataclient "github.com/bnb-chain/greenfield-storage-provider/service/metadata/client" @@ -95,12 +96,12 @@ func NewGatewayService(cfg *GatewayConfig) (*Gateway, error) { return gateway, nil } -// Name implement the lifecycle interface +// Name return the descriptions of gateway service func (gateway *Gateway) Name() string { return model.GatewayService } -// Start implement the lifecycle interface +// Start gateway service func (gateway *Gateway) Start(ctx context.Context) error { if gateway.running.Swap(true) == true { return errors.New("gateway has started") @@ -109,9 +110,12 @@ func (gateway *Gateway) Start(ctx context.Context) error { return nil } -// Serve starts http service. +// Serve starts http server. func (gateway *Gateway) serve() { router := mux.NewRouter().SkipClean(true) + if metrics.GetMetrics().Enabled() { + router.Use(metrics.DefaultHTTPServerMetrics.InstrumentationHandler) + } gateway.registerHandler(router) server := &http.Server{ Addr: gateway.config.HTTPAddress, @@ -124,7 +128,7 @@ func (gateway *Gateway) serve() { } } -// Stop implement the lifecycle interface +// Stop gateway service func (gateway *Gateway) Stop(ctx context.Context) error { if gateway.running.Swap(false) == false { return errors.New("gateway has stopped") diff --git a/service/uploader/client/uploader_client.go b/service/uploader/client/uploader_client.go index dde3cbb4e..63e4dda75 100644 --- a/service/uploader/client/uploader_client.go +++ b/service/uploader/client/uploader_client.go @@ -3,13 +3,15 @@ package client import ( "context" - "github.com/bnb-chain/greenfield-storage-provider/model" + openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics" servicetypes "github.com/bnb-chain/greenfield-storage-provider/service/types" - types "github.com/bnb-chain/greenfield-storage-provider/service/uploader/types" + "github.com/bnb-chain/greenfield-storage-provider/service/uploader/types" ) // UploaderClient is an uploader gRPC service client wrapper @@ -20,10 +22,15 @@ type UploaderClient struct { // NewUploaderClient return an UploaderClient instance func NewUploaderClient(address string) (*UploaderClient, error) { - conn, err := grpc.DialContext(context.Background(), address, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize)), - grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize))) + var options []grpc.DialOption + options = append(options, grpc.WithTransportCredentials(insecure.NewCredentials())) + options = append(options, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(model.MaxCallMsgSize))) + options = append(options, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(model.MaxCallMsgSize))) + if metrics.GetMetrics().Enabled() { + options = append(options, grpc.WithChainUnaryInterceptor(openmetrics.UnaryClientInterceptor(metrics.DefaultGRPCClientMetrics))) + options = append(options, grpc.WithChainStreamInterceptor(openmetrics.StreamClientInterceptor(metrics.DefaultGRPCClientMetrics))) + } + conn, err := grpc.DialContext(context.Background(), address, options...) if err != nil { log.Errorw("fail to invoke uploader service client", "error", err) return nil, err @@ -35,7 +42,7 @@ func NewUploaderClient(address string) (*UploaderClient, error) { return client, nil } -// Close the uploader gPRC client connection +// Close the uploader gRPC client connection func (client *UploaderClient) Close() error { return client.conn.Close() } diff --git a/service/uploader/uploader.go b/service/uploader/uploader.go index fc67f93d7..b997cf809 100644 --- a/service/uploader/uploader.go +++ b/service/uploader/uploader.go @@ -3,19 +3,25 @@ package uploader import ( "context" "net" + "runtime/debug" - "github.com/bnb-chain/greenfield-storage-provider/store/sqldb" + openmetrics "github.com/grpc-ecosystem/go-grpc-middleware/providers/openmetrics/v2" + grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" lru "github.com/hashicorp/golang-lru" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/pkg/metrics" signerclient "github.com/bnb-chain/greenfield-storage-provider/service/signer/client" tasknodeclient "github.com/bnb-chain/greenfield-storage-provider/service/tasknode/client" "github.com/bnb-chain/greenfield-storage-provider/service/uploader/types" psclient "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/client" + "github.com/bnb-chain/greenfield-storage-provider/store/sqldb" ) var _ lifecycle.Service = &Uploader{} @@ -98,11 +104,24 @@ func (uploader *Uploader) serve(errCh chan error) { return } - grpcServer := grpc.NewServer(grpc.MaxRecvMsgSize(model.MaxCallMsgSize), grpc.MaxSendMsgSize(model.MaxCallMsgSize)) - types.RegisterUploaderServiceServer(grpcServer, uploader) - uploader.grpcServer = grpcServer - reflection.Register(grpcServer) - if err := grpcServer.Serve(lis); err != nil { + gRPCPanicRecoveryHandler := func(p interface{}) (err error) { + metrics.PanicsTotal.WithLabelValues().Inc() + log.Errorw("recovered from panic", "panic", p, "stack", debug.Stack()) + return status.Errorf(codes.Internal, "%s", p) + } + + var options []grpc.ServerOption + options = append(options, grpc.MaxRecvMsgSize(model.MaxCallMsgSize)) + options = append(options, grpc.MaxSendMsgSize(model.MaxCallMsgSize)) + if metrics.GetMetrics().Enabled() { + options = append(options, grpc.ChainUnaryInterceptor(openmetrics.UnaryServerInterceptor(metrics.DefaultGRPCServerMetrics), + grpcrecovery.UnaryServerInterceptor(grpcrecovery.WithRecoveryHandler(gRPCPanicRecoveryHandler)))) + options = append(options, grpc.ChainStreamInterceptor(openmetrics.StreamServerInterceptor(metrics.DefaultGRPCServerMetrics))) + } + uploader.grpcServer = grpc.NewServer(options...) + types.RegisterUploaderServiceServer(uploader.grpcServer, uploader) + reflection.Register(uploader.grpcServer) + if err := uploader.grpcServer.Serve(lis); err != nil { log.Errorw("failed to start grpc server", "error", err) return }