From b39ff5b07e3118057d84f43a5e6fe16bb8842eb6 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Fri, 3 Mar 2023 21:20:13 +0800 Subject: [PATCH 01/10] chore: add spdb address in env var --- cmd/utils/flags.go | 7 ++++--- model/const.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 879f41edf..54126d857 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -38,9 +38,10 @@ var ( EnvVars: []string{model.SpDBPasswd}, } DBAddressFlag = &cli.StringFlag{ - Name: "db.address", - Usage: "DB address", - Value: "localhost:3306", + Name: "db.address", + Usage: "DB address", + EnvVars: []string{model.SpDBAddress}, + Value: "localhost:3306", } LogLevelFlag = &cli.StringFlag{ Name: "log.level", diff --git a/model/const.go b/model/const.go index 800f4a893..9fbd51df5 100644 --- a/model/const.go +++ b/model/const.go @@ -82,6 +82,8 @@ const ( SpDBUser = "SP_DB_USER" // SpDBPasswd defines env variable name for sp db user passwd SpDBPasswd = "SP_DB_PASSWORD" + // SpDBAddress defines env variable name for sp db address + SpDBAddress = "SP_DB_Address" // SpOperatorAddress defines env variable name for sp operator address SpOperatorAddress = "SP_OPERATOR_PUB_KEY" From a3919f045ecd7eb344ba65ec8578fb18ac03455c Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 18:50:11 +0800 Subject: [PATCH 02/10] fix: stone node replicate idx --- service/stonenode/stone_node.go | 12 ++++++------ service/stonenode/stone_node_service.go | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/service/stonenode/stone_node.go b/service/stonenode/stone_node.go index 186a7a1a5..86be408fc 100644 --- a/service/stonenode/stone_node.go +++ b/service/stonenode/stone_node.go @@ -113,8 +113,8 @@ func (node *StoneNode) EncodeReplicateSegments(ctx context.Context, objectID uin if err != nil { return } - for i := 0; i < replicates; i++ { - data = append(data, make([][]byte, int(segments))) + for i := 0; i < int(segments); i++ { + data = append(data, make([][]byte, replicates)) } log.Debugw("start to encode payload", "object_id", objectID, "segment_count", segments, "replicas", replicates, "redundancy_type", rType) @@ -137,12 +137,12 @@ func (node *StoneNode) EncodeReplicateSegments(ctx context.Context, objectID uin errCh <- err return } - for idx, ec := range encodeData { - data[idx][segIdx] = ec + for rIdx, ecData := range encodeData { + data[segIdx][rIdx] = ecData } } else { - for idx := 0; idx < replicates; idx++ { - data[idx][segIdx] = segmentData + for rIdx := 0; rIdx < replicates; rIdx++ { + data[segIdx][rIdx] = segmentData } } log.Debugw("finish to encode payload", "object_id", objectID, "segment_idx", segIdx) diff --git a/service/stonenode/stone_node_service.go b/service/stonenode/stone_node_service.go index 38263cd58..805c54b8f 100644 --- a/service/stonenode/stone_node_service.go +++ b/service/stonenode/stone_node_service.go @@ -101,7 +101,6 @@ func (node *StoneNode) AsyncReplicateObject(req *types.ReplicateObjectRequest) ( spList = spList[1:] return sp, nil } - processInfo.SegmentInfos = make([]*servicetypes.SegmentInfo, replicates) var done int64 errCh := make(chan error, 10) @@ -117,7 +116,7 @@ func (node *StoneNode) AsyncReplicateObject(req *types.ReplicateObjectRequest) ( } var data [][]byte for idx := 0; idx < int(segments); idx++ { - data = append(data, replicateData[rIdx][idx]) + data = append(data, replicateData[idx][rIdx]) } gatewayClient, err := gatewayclient.NewGatewayClient(sp.GetEndpoint()) if err != nil { From 9e02599a9bf5056bc39fdc0d4d51deb3d9704767 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 22:07:01 +0800 Subject: [PATCH 03/10] refactor: the entry and config --- cmd/conf/conf.go | 18 +-- cmd/storage_provider/init.go | 91 +++++++---- cmd/storage_provider/main.go | 62 +++----- cmd/storage_provider/version.go | 18 +++ cmd/utils/db.go | 22 +++ cmd/utils/flags.go | 11 +- config/config.go | 118 +++++++++++---- config/subconfig.go | 167 +++++++++++++++++++++ model/const.go | 2 + pkg/greenfield/greenfield_config.go | 10 -- service/challenge/challenge.go | 1 - service/challenge/challenge_config.go | 7 - service/downloader/downloader_config.go | 8 - service/gateway/gateway_config.go | 13 -- service/metadata/metadata_config.go | 5 - service/stonenode/stone_node_config.go | 10 -- service/syncer/syncer_config.go | 9 -- service/uploader/uploader_config.go | 9 -- store/config/db_config.go | 8 - store/piecestore/storage/storage_config.go | 13 -- store/sqldb/store.go | 9 +- 21 files changed, 402 insertions(+), 209 deletions(-) create mode 100644 cmd/utils/db.go create mode 100644 config/subconfig.go diff --git a/cmd/conf/conf.go b/cmd/conf/conf.go index e00a4bd81..5f8bdcf2b 100644 --- a/cmd/conf/conf.go +++ b/cmd/conf/conf.go @@ -5,7 +5,6 @@ import ( "github.com/bnb-chain/greenfield-storage-provider/cmd/utils" "github.com/bnb-chain/greenfield-storage-provider/config" - storeconf "github.com/bnb-chain/greenfield-storage-provider/store/config" ) var ConfigDumpCmd = &cli.Command{ @@ -42,13 +41,14 @@ SP_DB_USER and SP_DB_PASSWORD`, // configUploadAction is the config.upload command. func configUploadAction(ctx *cli.Context) error { - _ = config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) - _ = &storeconf.SQLDBConfig{ - User: ctx.String(utils.DBUserFlag.Name), - Passwd: ctx.String(utils.DBPasswordFlag.Name), - Address: ctx.String(utils.DBAddressFlag.Name), - Database: "job_db", + cfg := config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) + cfgBytes, err := cfg.JsonMarshal() + if err != nil { + return err } - // TODO:: new sp db and upload config - return nil + spDB, err := utils.MakeSPDB(ctx) + if err != nil { + return err + } + return spDB.SetAllServiceConfigs("default", string(cfgBytes)) } diff --git a/cmd/storage_provider/init.go b/cmd/storage_provider/init.go index dc2248226..ad4e9ec12 100644 --- a/cmd/storage_provider/init.go +++ b/cmd/storage_provider/init.go @@ -4,85 +4,112 @@ import ( "context" "fmt" - "github.com/bnb-chain/greenfield-storage-provider/config" - "github.com/bnb-chain/greenfield-storage-provider/model" - "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" - "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/cmd/utils" "github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer" "github.com/bnb-chain/greenfield-storage-provider/service/challenge" "github.com/bnb-chain/greenfield-storage-provider/service/downloader" - "github.com/bnb-chain/greenfield-storage-provider/service/gateway" "github.com/bnb-chain/greenfield-storage-provider/service/metadata" "github.com/bnb-chain/greenfield-storage-provider/service/signer" "github.com/bnb-chain/greenfield-storage-provider/service/stonenode" "github.com/bnb-chain/greenfield-storage-provider/service/syncer" + "github.com/urfave/cli/v2" + + "github.com/bnb-chain/greenfield-storage-provider/config" + "github.com/bnb-chain/greenfield-storage-provider/model" + "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" + "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/service/gateway" "github.com/bnb-chain/greenfield-storage-provider/service/uploader" ) +// initLog init global log level and log path. +func initLog(ctx *cli.Context, cfg *config.StorageProviderConfig) error { + if cfg.LogCfg == nil { + cfg.LogCfg = config.DefaultLogConfig + } + if ctx.IsSet(utils.LogLevelFlag.Name) { + cfg.LogCfg.Level = ctx.String(utils.LogLevelFlag.Name) + } + if ctx.IsSet(utils.LogPathFlag.Name) { + cfg.LogCfg.Path = ctx.String(utils.LogPathFlag.Name) + } + logLevel, err := log.ParseLevel(cfg.LogCfg.Level) + if err != nil { + return err + } + log.Init(logLevel, cfg.LogCfg.Path) + return nil +} + // initService init service instance by name and config. func initService(serviceName string, cfg *config.StorageProviderConfig) (server lifecycle.Service, err error) { switch serviceName { case model.GatewayService: - if cfg.GatewayCfg == nil { - cfg.GatewayCfg = config.DefaultStorageProviderConfig.GatewayCfg + gCfg, err := cfg.MakeGatewayConfig() + if err != nil { + return nil, err } - server, err = gateway.NewGatewayService(cfg.GatewayCfg) + server, err = gateway.NewGatewayService(gCfg) if err != nil { return nil, err } case model.UploaderService: - if cfg.UploaderCfg == nil { - cfg.UploaderCfg = config.DefaultStorageProviderConfig.UploaderCfg + uCfg, err := cfg.MakeUploaderConfig() + if err != nil { + return nil, err } - server, err = uploader.NewUploaderService(cfg.UploaderCfg) + server, err = uploader.NewUploaderService(uCfg) if err != nil { return nil, err } case model.DownloaderService: - if cfg.DownloaderCfg == nil { - cfg.DownloaderCfg = config.DefaultStorageProviderConfig.DownloaderCfg - } - server, err = downloader.NewDownloaderService(cfg.DownloaderCfg) + dCfg, err := cfg.MakeDownloaderConfig() if err != nil { return nil, err } - case model.StoneNodeService: - if cfg.StoneNodeCfg == nil { - cfg.StoneNodeCfg = config.DefaultStorageProviderConfig.StoneNodeCfg - } - server, err = stonenode.NewStoneNodeService(cfg.StoneNodeCfg) + server, err = downloader.NewDownloaderService(dCfg) if err != nil { return nil, err } case model.SyncerService: - if cfg.SyncerCfg == nil { - cfg.SyncerCfg = config.DefaultStorageProviderConfig.SyncerCfg + sCfg, err := cfg.MakeSyncerConfig() + if err != nil { + return nil, err } - server, err = syncer.NewSyncerService(cfg.SyncerCfg) + server, err = syncer.NewSyncerService(sCfg) if err != nil { return nil, err } case model.ChallengeService: - if cfg.ChallengeCfg == nil { - cfg.ChallengeCfg = config.DefaultStorageProviderConfig.ChallengeCfg + cCfg, err := cfg.MakeChallengeConfig() + if err != nil { + return nil, err } - server, err = challenge.NewChallengeService(cfg.ChallengeCfg) + server, err = challenge.NewChallengeService(cCfg) if err != nil { return nil, err } case model.SignerService: - if cfg.SignerCfg == nil { - cfg.SignerCfg = config.DefaultStorageProviderConfig.SignerCfg + sCfg, _ := cfg.MakeSignerConfig() + server, err = signer.NewSignerServer(sCfg) + if err != nil { + return nil, err } - server, err = signer.NewSignerServer(cfg.SignerCfg) + case model.StoneNodeService: + snCfg, err := cfg.MakeStoneNodeConfig() + if err != nil { + return nil, err + } + server, err = stonenode.NewStoneNodeService(snCfg) if err != nil { return nil, err } case model.MetadataService: - if cfg.MetadataCfg == nil { - cfg.MetadataCfg = config.DefaultStorageProviderConfig.MetadataCfg + mCfg, err := cfg.MakeMetadataServiceConfig() + if err != nil { + return nil, err } - server, err = metadata.NewMetadataService(cfg.MetadataCfg, context.Background()) + server, err = metadata.NewMetadataService(mCfg, context.Background()) if err != nil { return nil, err } diff --git a/cmd/storage_provider/main.go b/cmd/storage_provider/main.go index df366e93a..0491ef932 100644 --- a/cmd/storage_provider/main.go +++ b/cmd/storage_provider/main.go @@ -4,18 +4,16 @@ import ( "context" "fmt" "os" - "strings" "syscall" - "github.com/bnb-chain/greenfield-storage-provider/model" - "github.com/bnb-chain/greenfield-storage-provider/util" + "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" "github.com/urfave/cli/v2" "github.com/bnb-chain/greenfield-storage-provider/cmd/conf" "github.com/bnb-chain/greenfield-storage-provider/cmd/utils" "github.com/bnb-chain/greenfield-storage-provider/config" - "github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle" "github.com/bnb-chain/greenfield-storage-provider/pkg/log" + "github.com/bnb-chain/greenfield-storage-provider/util" ) var ( @@ -28,7 +26,6 @@ var app *cli.App var ( // flags that configure the storage provider spFlags = []cli.Flag{ - utils.VersionFlag, utils.ConfigFileFlag, utils.ConfigRemoteFlag, utils.ServerFlag, @@ -52,6 +49,7 @@ func init() { conf.ConfigDumpCmd, conf.ConfigUploadCmd, // miscellaneous commands + VersionCmd, utils.ListServiceCmd, } } @@ -65,32 +63,32 @@ func main() { // makeConfig loads the configuration and creates the storage provider backend. func makeConfig(ctx *cli.Context) (*config.StorageProviderConfig, error) { - cfg := &config.StorageProviderConfig{} - var services []string - if ctx.IsSet(utils.ServerFlag.Name) { - services = util.SplitByComma(ctx.String(utils.ServerFlag.Name)) - for _, service := range services { - if _, ok := model.SpServiceDesc[strings.ToLower(service)]; !ok { - return nil, fmt.Errorf("invalid service %s", service) - } - } - } + // load config from remote db or local config file + cfg := config.DefaultStorageProviderConfig if ctx.IsSet(utils.ConfigRemoteFlag.Name) { - //&storeconf.SQLDBConfig{ - // User: ctx.String(utils.DBUserFlag.Name), - // Passwd: ctx.String(utils.DBPasswordFlag.Name), - // Address: ctx.String(utils.DBAddressFlag.Name), - // Database: "job_db", - //} - // TODO:: new sp db load config by services + spDB, err := utils.MakeSPDB(ctx) + if err != nil { + return nil, err + } + _, cfgBytes, err := spDB.GetAllServiceConfigs() + if err != nil { + return nil, err + } + if err := cfg.JsonUnMarshal([]byte(cfgBytes)); err != nil { + return nil, err + } } else if ctx.IsSet(utils.ConfigFileFlag.Name) { cfg = config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) - } else { - cfg = config.DefaultStorageProviderConfig } + // override the services to be started by flag if ctx.IsSet(utils.ServerFlag.Name) { + services := util.SplitByComma(ctx.String(utils.ServerFlag.Name)) cfg.Service = services } + // init log + if err := initLog(ctx, cfg); err != nil { + return nil, err + } return cfg, nil } @@ -98,33 +96,23 @@ func makeConfig(ctx *cli.Context) (*config.StorageProviderConfig, error) { // It uses default config to run storage provider services based on the command line arguments // and runs it in blocking mode, waiting for it to be shut down. func storageProvider(ctx *cli.Context) error { - if ctx.IsSet(utils.VersionFlag.Name) { - fmt.Print(DumpLogo() + "\n" + DumpVersion()) - return nil - } - logLevel, err := log.ParseLevel(ctx.String(utils.LogLevelFlag.Name)) - if err != nil { - return err - } - log.Init(logLevel, ctx.String(utils.LogPathFlag.Name)) - cfg, err := makeConfig(ctx) if err != nil { return err } slc := lifecycle.NewServiceLifecycle() for _, serviceName := range cfg.Service { - // 1. init service instance. + // init service instance. service, err := initService(serviceName, cfg) if err != nil { log.Errorw("failed to init service", "service", serviceName, "error", err) os.Exit(1) } log.Debugw("success to init service ", "service", serviceName) - // 2. register service to lifecycle. + // register service to lifecycle. slc.RegisterServices(service) } - // 3. start all services and listen os signals. + // start all services and listen os signals. slcCtx := context.Background() slc.Signals(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP).StartServices(slcCtx).Wait(slcCtx) return nil diff --git a/cmd/storage_provider/version.go b/cmd/storage_provider/version.go index 1fef8f2ad..49a554675 100644 --- a/cmd/storage_provider/version.go +++ b/cmd/storage_provider/version.go @@ -3,6 +3,8 @@ package main import ( "fmt" "runtime" + + "github.com/urfave/cli/v2" ) const ( @@ -39,3 +41,19 @@ func DumpVersion() string { CommitID, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildTime) } + +var VersionCmd = &cli.Command{ + Action: versionAction, + Name: "version", + Aliases: []string{"v"}, + Usage: "Print version information", + Category: "MISCELLANEOUS COMMANDS", + Description: ` +The output of this command is supposed to be machine-readable. +`, +} + +func versionAction(ctx *cli.Context) error { + fmt.Print(DumpLogo() + "\n" + DumpVersion()) + return nil +} diff --git a/cmd/utils/db.go b/cmd/utils/db.go new file mode 100644 index 000000000..b7e337dcd --- /dev/null +++ b/cmd/utils/db.go @@ -0,0 +1,22 @@ +package utils + +import ( + "github.com/bnb-chain/greenfield-storage-provider/config" + "github.com/bnb-chain/greenfield-storage-provider/store/sqldb" + "github.com/urfave/cli/v2" +) + +// MakeSPDB return sp db instance +func MakeSPDB(ctx *cli.Context) (*sqldb.SpDBImpl, error) { + spDBCfg := config.DefaultSQLDBConfig + if ctx.IsSet(ctx.String(DBUserFlag.Name)) { + spDBCfg.User = ctx.String(DBUserFlag.Name) + } + if ctx.IsSet(ctx.String(DBPasswordFlag.Name)) { + spDBCfg.Passwd = ctx.String(DBPasswordFlag.Name) + } + if ctx.IsSet(ctx.String(DBAddressFlag.Name)) { + spDBCfg.Address = ctx.String(DBAddressFlag.Name) + } + return sqldb.NewSpDB(spDBCfg) +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 54126d857..0ffb3b451 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -7,11 +7,6 @@ import ( ) var ( - VersionFlag = &cli.BoolFlag{ - Name: "version", - Aliases: []string{"v"}, - Usage: "Show the storage provider version information", - } ConfigFileFlag = &cli.StringFlag{ Name: "config", Aliases: []string{"c"}, @@ -19,8 +14,10 @@ var ( Value: "./config.toml", } ConfigRemoteFlag = &cli.BoolFlag{ - Name: "config.remote", - Usage: "Flag load config from remote db", + Name: "config.remote", + Usage: `Flag load config from remote db, if 'config.remote' be set, + the db.user, db.password and db.address flags are needed, + otherwise use default value`, } ServerFlag = &cli.StringFlag{ Name: "server", diff --git a/config/config.go b/config/config.go index 9785e42ce..8e5dbd42b 100644 --- a/config/config.go +++ b/config/config.go @@ -2,49 +2,50 @@ package config import ( "bufio" + "encoding/hex" + "encoding/json" "os" + "github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer" + "github.com/bnb-chain/greenfield-storage-provider/service/signer" tomlconfig "github.com/forbole/juno/v4/cmd/migrate/toml" "github.com/naoina/toml" "github.com/bnb-chain/greenfield-storage-provider/model" - "github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer" - "github.com/bnb-chain/greenfield-storage-provider/service/challenge" - "github.com/bnb-chain/greenfield-storage-provider/service/downloader" - "github.com/bnb-chain/greenfield-storage-provider/service/gateway" - "github.com/bnb-chain/greenfield-storage-provider/service/metadata" - "github.com/bnb-chain/greenfield-storage-provider/service/signer" - "github.com/bnb-chain/greenfield-storage-provider/service/stonenode" - "github.com/bnb-chain/greenfield-storage-provider/service/syncer" - "github.com/bnb-chain/greenfield-storage-provider/service/uploader" + gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" + "github.com/bnb-chain/greenfield-storage-provider/store/config" + storeconfig "github.com/bnb-chain/greenfield-storage-provider/store/config" + "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" "github.com/bnb-chain/greenfield-storage-provider/util" ) // StorageProviderConfig defines the configuration of storage provider type StorageProviderConfig struct { - Service []string - GatewayCfg *gateway.GatewayConfig - UploaderCfg *uploader.UploaderConfig - DownloaderCfg *downloader.DownloaderConfig - ChallengeCfg *challenge.ChallengeConfig - StoneNodeCfg *stonenode.StoneNodeConfig - SyncerCfg *syncer.SyncerConfig - SignerCfg *signer.SignerConfig - MetadataCfg *metadata.MetadataConfig - BlockSyncerCfg *tomlconfig.TomlConfig + Service []string + SpOperatorAddress string + Domain string + HTTPAddress map[string]string + GRPCAddress map[string]string + SpDBConfig *config.SQLDBConfig + PieceStoreConfig *storage.PieceStoreConfig + ChainConfig *gnfd.GreenfieldChainConfig + SignerCfg *signer.SignerConfig + BlockSyncerCfg *tomlconfig.TomlConfig + LogCfg *LogConfig +} + +// JsonMarshal marshal the StorageProviderConfig to json format +func (cfg *StorageProviderConfig) JsonMarshal() ([]byte, error) { + return json.Marshal(cfg) +} + +// JsonUnMarshal unmarshal bytes to StorageProviderConfig struct +func (cfg *StorageProviderConfig) JsonUnMarshal(jsonBytes []byte) error { + return json.Unmarshal(jsonBytes, cfg) } // DefaultStorageProviderConfig defines the default configuration of storage provider services var DefaultStorageProviderConfig = &StorageProviderConfig{ - GatewayCfg: gateway.DefaultGatewayConfig, - UploaderCfg: uploader.DefaultUploaderConfig, - DownloaderCfg: downloader.DefaultDownloaderConfig, - ChallengeCfg: challenge.DefaultChallengeConfig, - StoneNodeCfg: stonenode.DefaultStoneNodeConfig, - SyncerCfg: syncer.DefaultSyncerConfig, - SignerCfg: signer.DefaultSignerChainConfig, - MetadataCfg: metadata.DefaultMetadataConfig, - BlockSyncerCfg: blocksyncer.DefaultBlockSyncerConfig, Service: []string{ model.GatewayService, model.UploaderService, @@ -54,6 +55,67 @@ var DefaultStorageProviderConfig = &StorageProviderConfig{ model.SyncerService, model.SignerService, }, + GRPCAddress: map[string]string{ + model.UploaderService: model.UploaderGRPCAddress, + model.DownloaderService: model.DownloaderGRPCAddress, + model.ChallengeService: model.ChallengeGRPCAddress, + model.SyncerService: model.SyncerGRPCAddress, + model.StoneNodeService: model.StoneNodeGRPCAddress, + model.SignerService: model.SignerGRPCAddress, + }, + HTTPAddress: map[string]string{ + model.GatewayService: model.GatewayHTTPAddress, + model.MetadataService: model.MetaDataServiceHTTPAddress, + }, + SpOperatorAddress: hex.EncodeToString([]byte("greenfield-storage-provider")), + Domain: "gnfd.nodereal.com", + SpDBConfig: DefaultSQLDBConfig, + PieceStoreConfig: DefaultPieceStoreConfig, + ChainConfig: DefaultGreenfieldChainConfig, + SignerCfg: signer.DefaultSignerChainConfig, + BlockSyncerCfg: blocksyncer.DefaultBlockSyncerConfig, + LogCfg: DefaultLogConfig, +} + +// DefaultSQLDBConfig defines the default configuration of SQL DB +var DefaultSQLDBConfig = &storeconfig.SQLDBConfig{ + User: "root", + Passwd: "test_pwd", + Address: "localhost:3306", + Database: "storage_provider_db", +} + +// DefaultPieceStoreConfig defines the default configuration of piece store +var DefaultPieceStoreConfig = &storage.PieceStoreConfig{ + Shards: 0, + Store: &storage.ObjectStorageConfig{ + Storage: "file", + BucketURL: "./data", + NoSignRequest: false, + MaxRetries: 5, + MinRetryDelay: 0, + TlsInsecureSkipVerify: false, + }, +} + +// DefaultGreenfieldChainConfig defines the default configuration of greenfield chain +var DefaultGreenfieldChainConfig = &gnfd.GreenfieldChainConfig{ + ChainID: model.GreenfieldChainID, + NodeAddr: []*gnfd.NodeConfig{{ + GreenfieldAddresses: []string{model.GreenfieldAddress}, + TendermintAddresses: []string{model.TendermintAddress}, + }}, +} + +type LogConfig struct { + Level string + Path string +} + +// DefaultLogConfig defines the default configuration of log +var DefaultLogConfig = &LogConfig{ + Level: "debug", + Path: "./gnfd-sp.log", } // LoadConfig loads the config file from path diff --git a/config/subconfig.go b/config/subconfig.go new file mode 100644 index 000000000..4ce2f50aa --- /dev/null +++ b/config/subconfig.go @@ -0,0 +1,167 @@ +package config + +import ( + "fmt" + + "github.com/bnb-chain/greenfield-storage-provider/model" + "github.com/bnb-chain/greenfield-storage-provider/service/challenge" + "github.com/bnb-chain/greenfield-storage-provider/service/downloader" + "github.com/bnb-chain/greenfield-storage-provider/service/gateway" + "github.com/bnb-chain/greenfield-storage-provider/service/metadata" + "github.com/bnb-chain/greenfield-storage-provider/service/signer" + "github.com/bnb-chain/greenfield-storage-provider/service/stonenode" + "github.com/bnb-chain/greenfield-storage-provider/service/syncer" + "github.com/bnb-chain/greenfield-storage-provider/service/uploader" +) + +// MakeGatewayConfig make gateway service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeGatewayConfig() (*gateway.GatewayConfig, error) { + gCfg := &gateway.GatewayConfig{ + SpOperatorAddress: cfg.SpOperatorAddress, + Domain: cfg.Domain, + ChainConfig: cfg.ChainConfig, + } + if _, ok := cfg.HTTPAddress[model.GatewayHTTPAddress]; ok { + gCfg.HTTPAddress = cfg.HTTPAddress[model.GatewayHTTPAddress] + } else { + return nil, fmt.Errorf("missing gateway HTTP address configuration") + } + if _, ok := cfg.GRPCAddress[model.UploaderGRPCAddress]; ok { + gCfg.UploaderServiceAddress = cfg.GRPCAddress[model.UploaderGRPCAddress] + } else { + return nil, fmt.Errorf("missing uploader gPRC address configuration for gateway service") + } + if _, ok := cfg.GRPCAddress[model.DownloaderGRPCAddress]; ok { + gCfg.DownloaderServiceAddress = cfg.GRPCAddress[model.DownloaderGRPCAddress] + } else { + return nil, fmt.Errorf("missing downloader gPRC address configuration for gateway service") + } + if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { + gCfg.SignerServiceAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + } else { + return nil, fmt.Errorf("missing signer gPRC address configuration for gateway service") + } + if _, ok := cfg.GRPCAddress[model.ChallengeGRPCAddress]; ok { + gCfg.ChallengeServiceAddress = cfg.GRPCAddress[model.ChallengeGRPCAddress] + } else { + return nil, fmt.Errorf("missing challenge gPRC address configuration for gateway service") + } + if _, ok := cfg.GRPCAddress[model.SyncerGRPCAddress]; ok { + gCfg.SyncerServiceAddress = cfg.GRPCAddress[model.SyncerGRPCAddress] + } else { + return nil, fmt.Errorf("missing syncer gPRC address configuration for gateway service") + } + return gCfg, nil +} + +// MakeUploaderConfig make uploader service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeUploaderConfig() (*uploader.UploaderConfig, error) { + uCfg := &uploader.UploaderConfig{ + SpDBConfig: cfg.SpDBConfig, + PieceStoreConfig: cfg.PieceStoreConfig, + } + if _, ok := cfg.GRPCAddress[model.UploaderService]; ok { + uCfg.GRPCAddress = cfg.GRPCAddress[model.UploaderService] + } else { + return nil, fmt.Errorf("missing uploader gPRC address configuration for uploader service") + } + if _, ok := cfg.GRPCAddress[model.SignerService]; ok { + uCfg.SignerGrpcAddress = cfg.GRPCAddress[model.SignerService] + } else { + return nil, fmt.Errorf("missing signer gPRC address configuration for uploader service") + } + if _, ok := cfg.GRPCAddress[model.StoneNodeGRPCAddress]; ok { + uCfg.StoneNodeGrpcAddress = cfg.GRPCAddress[model.StoneNodeGRPCAddress] + } else { + return nil, fmt.Errorf("missing stone node gPRC address configuration for uploader service") + } + return uCfg, nil +} + +// MakeDownloaderConfig make downloader service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeDownloaderConfig() (*downloader.DownloaderConfig, error) { + dCfg := &downloader.DownloaderConfig{ + SpDBConfig: cfg.SpDBConfig, + ChainConfig: cfg.ChainConfig, + PieceStoreConfig: cfg.PieceStoreConfig, + } + if _, ok := cfg.GRPCAddress[model.DownloaderService]; ok { + dCfg.GRPCAddress = cfg.GRPCAddress[model.DownloaderService] + } else { + return nil, fmt.Errorf("missing downloader gPRC address configuration for downloader service") + } + return dCfg, nil +} + +// MakeSyncerConfig make syncer service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeSyncerConfig() (*syncer.SyncerConfig, error) { + sCfg := &syncer.SyncerConfig{ + SpOperatorAddress: cfg.SpOperatorAddress, + SpDBConfig: cfg.SpDBConfig, + PieceStoreConfig: cfg.PieceStoreConfig, + } + if _, ok := cfg.GRPCAddress[model.SyncerService]; ok { + sCfg.GRPCAddress = cfg.GRPCAddress[model.SyncerService] + } else { + return nil, fmt.Errorf("missing syncer gPRC address configuration for syncer service") + } + if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { + sCfg.SignerGRPCAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + } else { + return nil, fmt.Errorf("missing signer gPRC address configuration for syncer service") + } + return sCfg, nil +} + +// MakeChallengeConfig make challenge service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeChallengeConfig() (*challenge.ChallengeConfig, error) { + cCfg := &challenge.ChallengeConfig{ + SpDBConfig: cfg.SpDBConfig, + PieceStoreConfig: cfg.PieceStoreConfig, + } + if _, ok := cfg.GRPCAddress[model.ChallengeService]; ok { + cCfg.GRPCAddress = cfg.GRPCAddress[model.ChallengeService] + } else { + return nil, fmt.Errorf("missing challenge gPRC address configuration for challenge service") + } + return cCfg, nil +} + +// MakeSignerConfig make singer service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeSignerConfig() (*signer.SignerConfig, error) { + return cfg.SignerCfg, nil +} + +// MakeStoneNodeConfig make stone node service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeStoneNodeConfig() (*stonenode.StoneNodeConfig, error) { + snCfg := &stonenode.StoneNodeConfig{ + SpOperatorAddress: cfg.SpOperatorAddress, + SpDBConfig: cfg.SpDBConfig, + PieceStoreConfig: cfg.PieceStoreConfig, + ChainConfig: cfg.ChainConfig, + } + if _, ok := cfg.GRPCAddress[model.StoneNodeService]; ok { + snCfg.GRPCAddress = cfg.GRPCAddress[model.StoneNodeService] + } else { + return nil, fmt.Errorf("missing stone node gPRC address configuration for stone node service") + } + if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { + snCfg.SignerGrpcAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + } else { + return nil, fmt.Errorf("missing signer gPRC address configuration for stone node service") + } + return snCfg, nil +} + +// MakeMetadataServiceConfig make meta data service config from StorageProviderConfig +func (cfg *StorageProviderConfig) MakeMetadataServiceConfig() (*metadata.MetadataConfig, error) { + mCfg := &metadata.MetadataConfig{ + SpDBConfig: cfg.SpDBConfig, + } + if _, ok := cfg.HTTPAddress[model.MetadataService]; ok { + mCfg.Address = cfg.HTTPAddress[model.MetadataService] + } else { + return nil, fmt.Errorf("missing meta data HTTP address configuration for mate data service") + } + return mCfg, nil +} diff --git a/model/const.go b/model/const.go index 9fbd51df5..da6c19b80 100644 --- a/model/const.go +++ b/model/const.go @@ -41,6 +41,8 @@ var SpServiceDesc = map[string]string{ const ( // GatewayHTTPAddress default HTTP address of gateway GatewayHTTPAddress = "localhost:9033" + // MetaDataServiceHTTPAddress default HTTP address of meta data service + MetaDataServiceHTTPAddress = "localhost:9733" // UploaderGRPCAddress default gRPC address of uploader UploaderGRPCAddress = "localhost:9133" // DownloaderGRPCAddress default gRPC address of downloader diff --git a/pkg/greenfield/greenfield_config.go b/pkg/greenfield/greenfield_config.go index accb7045b..d9c6cdb88 100644 --- a/pkg/greenfield/greenfield_config.go +++ b/pkg/greenfield/greenfield_config.go @@ -1,7 +1,5 @@ package greenfield -import "github.com/bnb-chain/greenfield-storage-provider/model" - type NodeConfig struct { GreenfieldAddresses []string TendermintAddresses []string @@ -11,11 +9,3 @@ type GreenfieldChainConfig struct { ChainID string NodeAddr []*NodeConfig } - -var DefaultGreenfieldChainConfig = &GreenfieldChainConfig{ - ChainID: "greenfield_9000-1741", - NodeAddr: []*NodeConfig{{ - GreenfieldAddresses: []string{model.GreenfieldAddress}, - TendermintAddresses: []string{model.TendermintAddress}, - }}, -} diff --git a/service/challenge/challenge.go b/service/challenge/challenge.go index 46580bcaf..c1e731222 100644 --- a/service/challenge/challenge.go +++ b/service/challenge/challenge.go @@ -29,7 +29,6 @@ func NewChallengeService(config *ChallengeConfig) (challenge *Challenge, err err if err != nil { return nil, err } - // TODO:: new sp db spDB, err := sqldb.NewSpDB(config.SpDBConfig) if err != nil { return nil, err diff --git a/service/challenge/challenge_config.go b/service/challenge/challenge_config.go index b7b5cb764..b99d6d828 100644 --- a/service/challenge/challenge_config.go +++ b/service/challenge/challenge_config.go @@ -1,7 +1,6 @@ package challenge import ( - "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" ) @@ -11,9 +10,3 @@ type ChallengeConfig struct { SpDBConfig *config.SQLDBConfig PieceStoreConfig *storage.PieceStoreConfig } - -var DefaultChallengeConfig = &ChallengeConfig{ - GRPCAddress: model.ChallengeGRPCAddress, - SpDBConfig: config.DefaultSQLDBConfig, - PieceStoreConfig: storage.DefaultPieceStoreConfig, -} diff --git a/service/downloader/downloader_config.go b/service/downloader/downloader_config.go index 2bdc47d5c..45b923c52 100644 --- a/service/downloader/downloader_config.go +++ b/service/downloader/downloader_config.go @@ -1,7 +1,6 @@ package downloader import ( - "github.com/bnb-chain/greenfield-storage-provider/model" gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" @@ -13,10 +12,3 @@ type DownloaderConfig struct { PieceStoreConfig *storage.PieceStoreConfig ChainConfig *gnfd.GreenfieldChainConfig } - -var DefaultDownloaderConfig = &DownloaderConfig{ - GRPCAddress: model.DownloaderGRPCAddress, - SpDBConfig: config.DefaultSQLDBConfig, - ChainConfig: gnfd.DefaultGreenfieldChainConfig, - PieceStoreConfig: storage.DefaultPieceStoreConfig, -} diff --git a/service/gateway/gateway_config.go b/service/gateway/gateway_config.go index e9ccaa016..8ea8b9729 100644 --- a/service/gateway/gateway_config.go +++ b/service/gateway/gateway_config.go @@ -1,7 +1,6 @@ package gateway import ( - "github.com/bnb-chain/greenfield-storage-provider/model" gnfd "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" ) @@ -16,15 +15,3 @@ type GatewayConfig struct { SyncerServiceAddress string ChainConfig *gnfd.GreenfieldChainConfig } - -var DefaultGatewayConfig = &GatewayConfig{ - SpOperatorAddress: model.SpOperatorAddress, - HTTPAddress: model.GatewayHTTPAddress, - Domain: "gnfd.nodereal.com", - UploaderServiceAddress: model.UploaderGRPCAddress, - DownloaderServiceAddress: model.DownloaderGRPCAddress, - SyncerServiceAddress: model.SyncerGRPCAddress, - SignerServiceAddress: model.SignerGRPCAddress, - ChallengeServiceAddress: model.ChallengeGRPCAddress, - ChainConfig: gnfd.DefaultGreenfieldChainConfig, -} diff --git a/service/metadata/metadata_config.go b/service/metadata/metadata_config.go index 8608aeb65..6b260efe4 100644 --- a/service/metadata/metadata_config.go +++ b/service/metadata/metadata_config.go @@ -8,8 +8,3 @@ type MetadataConfig struct { Address string SpDBConfig *config.SQLDBConfig } - -var DefaultMetadataConfig = &MetadataConfig{ - Address: "127.0.0.1:9733", - SpDBConfig: config.DefaultSQLDBConfig, -} diff --git a/service/stonenode/stone_node_config.go b/service/stonenode/stone_node_config.go index 3c98c024f..0cb5bfe98 100644 --- a/service/stonenode/stone_node_config.go +++ b/service/stonenode/stone_node_config.go @@ -1,7 +1,6 @@ package stonenode import ( - "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/pkg/greenfield" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" @@ -15,12 +14,3 @@ type StoneNodeConfig struct { PieceStoreConfig *storage.PieceStoreConfig ChainConfig *greenfield.GreenfieldChainConfig } - -var DefaultStoneNodeConfig = &StoneNodeConfig{ - SpOperatorAddress: model.SpOperatorAddress, - GRPCAddress: model.StoneNodeGRPCAddress, - SignerGrpcAddress: model.SignerGRPCAddress, - SpDBConfig: config.DefaultSQLDBConfig, - PieceStoreConfig: storage.DefaultPieceStoreConfig, - ChainConfig: greenfield.DefaultGreenfieldChainConfig, -} diff --git a/service/syncer/syncer_config.go b/service/syncer/syncer_config.go index e282e496c..a5962690e 100644 --- a/service/syncer/syncer_config.go +++ b/service/syncer/syncer_config.go @@ -1,7 +1,6 @@ package syncer import ( - "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" ) @@ -13,11 +12,3 @@ type SyncerConfig struct { SpDBConfig *config.SQLDBConfig PieceStoreConfig *storage.PieceStoreConfig } - -var DefaultSyncerConfig = &SyncerConfig{ - SpOperatorAddress: model.SpOperatorAddress, - GRPCAddress: model.SyncerGRPCAddress, - SignerGRPCAddress: model.SignerGRPCAddress, - SpDBConfig: config.DefaultSQLDBConfig, - PieceStoreConfig: storage.DefaultPieceStoreConfig, -} diff --git a/service/uploader/uploader_config.go b/service/uploader/uploader_config.go index 6acb30e0a..c48c9032a 100644 --- a/service/uploader/uploader_config.go +++ b/service/uploader/uploader_config.go @@ -1,7 +1,6 @@ package uploader import ( - "github.com/bnb-chain/greenfield-storage-provider/model" "github.com/bnb-chain/greenfield-storage-provider/store/config" "github.com/bnb-chain/greenfield-storage-provider/store/piecestore/storage" ) @@ -13,11 +12,3 @@ type UploaderConfig struct { SpDBConfig *config.SQLDBConfig PieceStoreConfig *storage.PieceStoreConfig } - -var DefaultUploaderConfig = &UploaderConfig{ - GRPCAddress: model.UploaderGRPCAddress, - SignerGrpcAddress: model.SyncerGRPCAddress, - StoneNodeGrpcAddress: model.StoneNodeGRPCAddress, - SpDBConfig: config.DefaultSQLDBConfig, - PieceStoreConfig: storage.DefaultPieceStoreConfig, -} diff --git a/store/config/db_config.go b/store/config/db_config.go index 5a0c72962..e1cb480a6 100644 --- a/store/config/db_config.go +++ b/store/config/db_config.go @@ -7,11 +7,3 @@ type SQLDBConfig struct { Address string Database string } - -// DefaultSQLDBConfig is default conf, Modify it according to the actual configuration. -var DefaultSQLDBConfig = &SQLDBConfig{ - User: "root", - Passwd: "test_pwd", - Address: "localhost:3306", - Database: "storage_provider_db", -} diff --git a/store/piecestore/storage/storage_config.go b/store/piecestore/storage/storage_config.go index bb8fb034e..1cf1b9fcc 100644 --- a/store/piecestore/storage/storage_config.go +++ b/store/piecestore/storage/storage_config.go @@ -6,19 +6,6 @@ type PieceStoreConfig struct { Store *ObjectStorageConfig // config of object storage } -// DefaultPieceStoreConfig if no config in config file, use this default config -var DefaultPieceStoreConfig = &PieceStoreConfig{ - Shards: 0, - Store: &ObjectStorageConfig{ - Storage: "file", - BucketURL: "./data", - NoSignRequest: false, - MaxRetries: 5, - MinRetryDelay: 0, - TlsInsecureSkipVerify: false, - }, -} - // ObjectStorageConfig object storage config type ObjectStorageConfig struct { Storage string // backend storage type (e.g. s3, file, memory) diff --git a/store/sqldb/store.go b/store/sqldb/store.go index ec15fcca8..1ed1682f6 100644 --- a/store/sqldb/store.go +++ b/store/sqldb/store.go @@ -21,7 +21,7 @@ type SpDBImpl struct { // NewSpDB return a database instance func NewSpDB(config *config.SQLDBConfig) (*SpDBImpl, error) { - loadDBConfigFromEnv(config) + LoadDBConfigFromEnv(config) db, err := InitDB(config) if err != nil { return nil, err @@ -75,12 +75,15 @@ func InitDB(config *config.SQLDBConfig) (*gorm.DB, error) { return db, nil } -// loadDBConfigFromEnv load db user and password from env vars -func loadDBConfigFromEnv(config *config.SQLDBConfig) { +// LoadDBConfigFromEnv load db user and password from env vars +func LoadDBConfigFromEnv(config *config.SQLDBConfig) { if val, ok := os.LookupEnv(model.SpDBUser); ok { config.User = val } if val, ok := os.LookupEnv(model.SpDBPasswd); ok { config.Passwd = val } + if val, ok := os.LookupEnv(model.SpDBAddress); ok { + config.Address = val + } } From 85fe0a80ec7ca33cb3b843b207b5adad51f9b3e6 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 22:33:31 +0800 Subject: [PATCH 04/10] fix: address and service map in config --- cmd/conf/conf.go | 2 +- cmd/utils/flags.go | 7 ++++--- config/subconfig.go | 34 +++++++++++++++++----------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/cmd/conf/conf.go b/cmd/conf/conf.go index 5f8bdcf2b..1689363e4 100644 --- a/cmd/conf/conf.go +++ b/cmd/conf/conf.go @@ -10,7 +10,7 @@ import ( var ConfigDumpCmd = &cli.Command{ Action: dumpConfigAction, Name: "config.dump", - Usage: "Dump default configuration to file for editing", + Usage: "Dump default configuration to the './config.toml' file for editing", Category: "CONFIG COMMANDS", Description: ` The config.dump command writes default configuration diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0ffb3b451..2712df3d2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -15,9 +15,10 @@ var ( } ConfigRemoteFlag = &cli.BoolFlag{ Name: "config.remote", - Usage: `Flag load config from remote db, if 'config.remote' be set, - the db.user, db.password and db.address flags are needed, - otherwise use default value`, + Usage: ` +Flag load config from remote db,if 'config.remote' be set, the db.user, db.password +and db.address flags are needed, otherwise use default value +`, } ServerFlag = &cli.StringFlag{ Name: "server", diff --git a/config/subconfig.go b/config/subconfig.go index 4ce2f50aa..b6eda4aa8 100644 --- a/config/subconfig.go +++ b/config/subconfig.go @@ -24,30 +24,30 @@ func (cfg *StorageProviderConfig) MakeGatewayConfig() (*gateway.GatewayConfig, e if _, ok := cfg.HTTPAddress[model.GatewayHTTPAddress]; ok { gCfg.HTTPAddress = cfg.HTTPAddress[model.GatewayHTTPAddress] } else { - return nil, fmt.Errorf("missing gateway HTTP address configuration") + return nil, fmt.Errorf("missing gateway HTTP address configuration for gateway service") } - if _, ok := cfg.GRPCAddress[model.UploaderGRPCAddress]; ok { - gCfg.UploaderServiceAddress = cfg.GRPCAddress[model.UploaderGRPCAddress] + if _, ok := cfg.GRPCAddress[model.UploaderService]; ok { + gCfg.UploaderServiceAddress = cfg.GRPCAddress[model.UploaderService] } else { return nil, fmt.Errorf("missing uploader gPRC address configuration for gateway service") } - if _, ok := cfg.GRPCAddress[model.DownloaderGRPCAddress]; ok { - gCfg.DownloaderServiceAddress = cfg.GRPCAddress[model.DownloaderGRPCAddress] + if _, ok := cfg.GRPCAddress[model.DownloaderService]; ok { + gCfg.DownloaderServiceAddress = cfg.GRPCAddress[model.DownloaderService] } else { return nil, fmt.Errorf("missing downloader gPRC address configuration for gateway service") } - if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { - gCfg.SignerServiceAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + if _, ok := cfg.GRPCAddress[model.SignerService]; ok { + gCfg.SignerServiceAddress = cfg.GRPCAddress[model.SignerService] } else { return nil, fmt.Errorf("missing signer gPRC address configuration for gateway service") } - if _, ok := cfg.GRPCAddress[model.ChallengeGRPCAddress]; ok { - gCfg.ChallengeServiceAddress = cfg.GRPCAddress[model.ChallengeGRPCAddress] + if _, ok := cfg.GRPCAddress[model.ChallengeService]; ok { + gCfg.ChallengeServiceAddress = cfg.GRPCAddress[model.ChallengeService] } else { return nil, fmt.Errorf("missing challenge gPRC address configuration for gateway service") } - if _, ok := cfg.GRPCAddress[model.SyncerGRPCAddress]; ok { - gCfg.SyncerServiceAddress = cfg.GRPCAddress[model.SyncerGRPCAddress] + if _, ok := cfg.GRPCAddress[model.SyncerService]; ok { + gCfg.SyncerServiceAddress = cfg.GRPCAddress[model.SyncerService] } else { return nil, fmt.Errorf("missing syncer gPRC address configuration for gateway service") } @@ -70,8 +70,8 @@ func (cfg *StorageProviderConfig) MakeUploaderConfig() (*uploader.UploaderConfig } else { return nil, fmt.Errorf("missing signer gPRC address configuration for uploader service") } - if _, ok := cfg.GRPCAddress[model.StoneNodeGRPCAddress]; ok { - uCfg.StoneNodeGrpcAddress = cfg.GRPCAddress[model.StoneNodeGRPCAddress] + if _, ok := cfg.GRPCAddress[model.StoneNodeService]; ok { + uCfg.StoneNodeGrpcAddress = cfg.GRPCAddress[model.StoneNodeService] } else { return nil, fmt.Errorf("missing stone node gPRC address configuration for uploader service") } @@ -105,8 +105,8 @@ func (cfg *StorageProviderConfig) MakeSyncerConfig() (*syncer.SyncerConfig, erro } else { return nil, fmt.Errorf("missing syncer gPRC address configuration for syncer service") } - if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { - sCfg.SignerGRPCAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + if _, ok := cfg.GRPCAddress[model.SignerService]; ok { + sCfg.SignerGRPCAddress = cfg.GRPCAddress[model.SignerService] } else { return nil, fmt.Errorf("missing signer gPRC address configuration for syncer service") } @@ -145,8 +145,8 @@ func (cfg *StorageProviderConfig) MakeStoneNodeConfig() (*stonenode.StoneNodeCon } else { return nil, fmt.Errorf("missing stone node gPRC address configuration for stone node service") } - if _, ok := cfg.GRPCAddress[model.SignerGRPCAddress]; ok { - snCfg.SignerGrpcAddress = cfg.GRPCAddress[model.SignerGRPCAddress] + if _, ok := cfg.GRPCAddress[model.SignerService]; ok { + snCfg.SignerGrpcAddress = cfg.GRPCAddress[model.SignerService] } else { return nil, fmt.Errorf("missing signer gPRC address configuration for stone node service") } From 86ef9a49bd876bb35e29691583a6c4ad22159140 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 22:42:11 +0800 Subject: [PATCH 05/10] chore:add log std output --- cmd/storage_provider/init.go | 3 +++ cmd/utils/flags.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/storage_provider/init.go b/cmd/storage_provider/init.go index ad4e9ec12..8af758270 100644 --- a/cmd/storage_provider/init.go +++ b/cmd/storage_provider/init.go @@ -33,6 +33,9 @@ func initLog(ctx *cli.Context, cfg *config.StorageProviderConfig) error { if ctx.IsSet(utils.LogPathFlag.Name) { cfg.LogCfg.Path = ctx.String(utils.LogPathFlag.Name) } + if ctx.IsSet(utils.LogStdOutputFlag.Name) { + cfg.LogCfg.Path = "" + } logLevel, err := log.ParseLevel(cfg.LogCfg.Level) if err != nil { return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 2712df3d2..625065dd8 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -15,10 +15,8 @@ var ( } ConfigRemoteFlag = &cli.BoolFlag{ Name: "config.remote", - Usage: ` -Flag load config from remote db,if 'config.remote' be set, the db.user, db.password -and db.address flags are needed, otherwise use default value -`, + Usage: "Flag load config from remote db,if 'config.remote' be set, the db.user, " + + "db.password and db.address flags are needed, otherwise use default value", } ServerFlag = &cli.StringFlag{ Name: "server", @@ -51,4 +49,8 @@ and db.address flags are needed, otherwise use default value Usage: "log path", Value: "./gnfd.log", } + LogStdOutputFlag = &cli.BoolFlag{ + Name: "log.std", + Usage: "log standard output", + } ) From ce768bb5ffed3754f27600784dd6d0fb0d3cb125 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 22:44:22 +0800 Subject: [PATCH 06/10] chore:add log std flag to mian app --- cmd/storage_provider/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/storage_provider/main.go b/cmd/storage_provider/main.go index 0491ef932..124bb4080 100644 --- a/cmd/storage_provider/main.go +++ b/cmd/storage_provider/main.go @@ -34,6 +34,7 @@ var ( utils.DBAddressFlag, utils.LogLevelFlag, utils.LogPathFlag, + utils.LogStdOutputFlag, } ) From 4ebfba9ceb995c709e5e84500bcc3e6d992d499f Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 22:54:46 +0800 Subject: [PATCH 07/10] chore:add toto list --- config/config.go | 1 + config/subconfig.go | 4 ++-- model/const.go | 12 +++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 8e5dbd42b..4c5dcdbff 100644 --- a/config/config.go +++ b/config/config.go @@ -114,6 +114,7 @@ type LogConfig struct { // DefaultLogConfig defines the default configuration of log var DefaultLogConfig = &LogConfig{ + // TODO:: change to info level after releasing Level: "debug", Path: "./gnfd-sp.log", } diff --git a/config/subconfig.go b/config/subconfig.go index b6eda4aa8..c482c889a 100644 --- a/config/subconfig.go +++ b/config/subconfig.go @@ -21,8 +21,8 @@ func (cfg *StorageProviderConfig) MakeGatewayConfig() (*gateway.GatewayConfig, e Domain: cfg.Domain, ChainConfig: cfg.ChainConfig, } - if _, ok := cfg.HTTPAddress[model.GatewayHTTPAddress]; ok { - gCfg.HTTPAddress = cfg.HTTPAddress[model.GatewayHTTPAddress] + if _, ok := cfg.HTTPAddress[model.GatewayService]; ok { + gCfg.HTTPAddress = cfg.HTTPAddress[model.GatewayService] } else { return nil, fmt.Errorf("missing gateway HTTP address configuration for gateway service") } diff --git a/model/const.go b/model/const.go index da6c19b80..67c911690 100644 --- a/model/const.go +++ b/model/const.go @@ -26,11 +26,13 @@ var ( // SpServiceDesc defines the service description in storage provider var SpServiceDesc = map[string]string{ - GatewayService: "Entrance for external user access", - UploaderService: "Upload object to the backend", - DownloaderService: "Download object from the backend and statistical read traffic", - ChallengeService: "Provides the ability to query the integrity hash", - StoneNodeService: "The smallest unit of background task execution", + GatewayService: "Entrance for external user access", + UploaderService: "Upload object to the backend", + DownloaderService: "Download object from the backend and statistical read traffic", + ChallengeService: "Provides the ability to query the integrity hash", + // TODO:: change other service name, maybe TaskService + StoneNodeService: "The smallest unit of background task execution", + // TODO:: change other service name, maybe ReplicateService SyncerService: "Receive object from other storage provider and store", SignerService: "Sign the transaction and broadcast to chain", MetadataService: "Provides the ability to query meta data", From ef8584ff49dcb66e088071aec48534bbeaf9e97a Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 23:28:33 +0800 Subject: [PATCH 08/10] chore: add db address env var --- cmd/conf/conf.go | 1 + cmd/storage_provider/main.go | 1 + cmd/utils/db.go | 3 +++ cmd/utils/flags.go | 8 +++++++- model/const.go | 4 +++- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/conf/conf.go b/cmd/conf/conf.go index 1689363e4..c561058ad 100644 --- a/cmd/conf/conf.go +++ b/cmd/conf/conf.go @@ -31,6 +31,7 @@ var ConfigUploadCmd = &cli.Command{ utils.DBUserFlag, utils.DBPasswordFlag, utils.DBAddressFlag, + utils.DBDataBaseFlag, }, Category: "CONFIG COMMANDS", Description: ` diff --git a/cmd/storage_provider/main.go b/cmd/storage_provider/main.go index 124bb4080..bd5e80ec0 100644 --- a/cmd/storage_provider/main.go +++ b/cmd/storage_provider/main.go @@ -32,6 +32,7 @@ var ( utils.DBUserFlag, utils.DBPasswordFlag, utils.DBAddressFlag, + utils.DBDataBaseFlag, utils.LogLevelFlag, utils.LogPathFlag, utils.LogStdOutputFlag, diff --git a/cmd/utils/db.go b/cmd/utils/db.go index b7e337dcd..98c43c8bb 100644 --- a/cmd/utils/db.go +++ b/cmd/utils/db.go @@ -18,5 +18,8 @@ func MakeSPDB(ctx *cli.Context) (*sqldb.SpDBImpl, error) { if ctx.IsSet(ctx.String(DBAddressFlag.Name)) { spDBCfg.Address = ctx.String(DBAddressFlag.Name) } + if ctx.IsSet(ctx.String(DBDataBaseFlag.Name)) { + spDBCfg.Database = ctx.String(DBDataBaseFlag.Name) + } return sqldb.NewSpDB(spDBCfg) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 625065dd8..a57e68a6f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -14,7 +14,7 @@ var ( Value: "./config.toml", } ConfigRemoteFlag = &cli.BoolFlag{ - Name: "config.remote", + Name: "configremote", Usage: "Flag load config from remote db,if 'config.remote' be set, the db.user, " + "db.password and db.address flags are needed, otherwise use default value", } @@ -39,6 +39,12 @@ var ( EnvVars: []string{model.SpDBAddress}, Value: "localhost:3306", } + DBDataBaseFlag = &cli.StringFlag{ + Name: "db.database", + Usage: "DB database", + EnvVars: []string{model.SpDBDataBase}, + Value: "localhost:3306", + } LogLevelFlag = &cli.StringFlag{ Name: "log.level", Usage: "log level", diff --git a/model/const.go b/model/const.go index 67c911690..4382b52f2 100644 --- a/model/const.go +++ b/model/const.go @@ -87,7 +87,9 @@ const ( // SpDBPasswd defines env variable name for sp db user passwd SpDBPasswd = "SP_DB_PASSWORD" // SpDBAddress defines env variable name for sp db address - SpDBAddress = "SP_DB_Address" + SpDBAddress = "SP_DB_ADDRESS" + // SpDBDataBase defines env variable name for sp db database + SpDBDataBase = "SP_DB_DATABASE" // SpOperatorAddress defines env variable name for sp operator address SpOperatorAddress = "SP_OPERATOR_PUB_KEY" From 6eb30c3bda1a29fc97a4345559944c35cd560517 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 23:42:22 +0800 Subject: [PATCH 09/10] fix: override the db database by env var --- cmd/conf/conf.go | 6 +++--- store/sqldb/store.go | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/conf/conf.go b/cmd/conf/conf.go index c561058ad..a549c23f5 100644 --- a/cmd/conf/conf.go +++ b/cmd/conf/conf.go @@ -42,12 +42,12 @@ SP_DB_USER and SP_DB_PASSWORD`, // configUploadAction is the config.upload command. func configUploadAction(ctx *cli.Context) error { - cfg := config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) - cfgBytes, err := cfg.JsonMarshal() + spDB, err := utils.MakeSPDB(ctx) if err != nil { return err } - spDB, err := utils.MakeSPDB(ctx) + cfg := config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name)) + cfgBytes, err := cfg.JsonMarshal() if err != nil { return err } diff --git a/store/sqldb/store.go b/store/sqldb/store.go index 1ed1682f6..156d5d9bc 100644 --- a/store/sqldb/store.go +++ b/store/sqldb/store.go @@ -86,4 +86,7 @@ func LoadDBConfigFromEnv(config *config.SQLDBConfig) { if val, ok := os.LookupEnv(model.SpDBAddress); ok { config.Address = val } + if val, ok := os.LookupEnv(model.SpDBDataBase); ok { + config.Database = val + } } From 75cd9cfd586af5bc85427f956723d75e42dfd1d9 Mon Sep 17 00:00:00 2001 From: joeylichang Date: Sat, 4 Mar 2023 23:58:18 +0800 Subject: [PATCH 10/10] fix: ci lint error --- pkg/stream/payload_stream.go | 1 - service/gateway/client/gateway_client.go | 4 ++++ service/stonenode/stone_node.go | 13 ++++--------- service/stonenode/stone_node_service.go | 9 +++------ 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pkg/stream/payload_stream.go b/pkg/stream/payload_stream.go index 1314f7258..322316e81 100644 --- a/pkg/stream/payload_stream.go +++ b/pkg/stream/payload_stream.go @@ -46,7 +46,6 @@ func (entry SegmentEntry) Error() error { type PayloadStream struct { objectID uint64 replicaIdx uint32 - segmentIdx uint32 segmentSize uint64 redundancyType storagetypes.RedundancyType entryCh chan *SegmentEntry diff --git a/service/gateway/client/gateway_client.go b/service/gateway/client/gateway_client.go index 5633bbbb1..deb1f5533 100644 --- a/service/gateway/client/gateway_client.go +++ b/service/gateway/client/gateway_client.go @@ -48,6 +48,10 @@ func (gatewayClient *GatewayClient) SyncPieceData(objectInfo *types.ObjectInfo, } req, err := http.NewRequest(http.MethodPut, gatewayClient.address+model.SyncerPath, bytes.NewReader(marshalPieceData)) + if err != nil { + log.Errorw("failed to new http request", "error", err) + return nil, nil, err + } req.Header.Add(model.GnfdObjectInfoHeader, marshalObjectInfo) req.Header.Add(model.GnfdReplicateIdxHeader, util.Uint32ToString(replicateIdx)) req.Header.Add(model.GnfdSegmentSizeHeader, util.Uint32ToString(segmentSize)) diff --git a/service/stonenode/stone_node.go b/service/stonenode/stone_node.go index 86be408fc..7ddbef063 100644 --- a/service/stonenode/stone_node.go +++ b/service/stonenode/stone_node.go @@ -137,9 +137,7 @@ func (node *StoneNode) EncodeReplicateSegments(ctx context.Context, objectID uin errCh <- err return } - for rIdx, ecData := range encodeData { - data[segIdx][rIdx] = ecData - } + copy(data[segIdx], encodeData) } else { for rIdx := 0; rIdx < replicates; rIdx++ { data[segIdx][rIdx] = segmentData @@ -152,11 +150,8 @@ func (node *StoneNode) EncodeReplicateSegments(ctx context.Context, objectID uin } }(segIdx) } - - for { - select { - case err = <-errCh: - return - } + for err = range errCh { + return } + return } diff --git a/service/stonenode/stone_node_service.go b/service/stonenode/stone_node_service.go index 805c54b8f..a39ca2f9c 100644 --- a/service/stonenode/stone_node_service.go +++ b/service/stonenode/stone_node_service.go @@ -59,7 +59,6 @@ func (node *StoneNode) AsyncReplicateObject(req *types.ReplicateObjectRequest) ( } node.spDB.UpdateJobState(objectInfo.Id.Uint64(), servicetypes.JobState_JOB_STATE_SEAL_OBJECT_DONE) log.CtxInfow(ctx, "seal object on chain", "success", success) - return }() params, err := node.spDB.GetStorageParams() @@ -162,12 +161,10 @@ func (node *StoneNode) AsyncReplicateObject(req *types.ReplicateObjectRequest) ( } }(rIdx) } - for { - select { - case err = <-errCh: - return - } + for err = range errCh { + return } + return } // QueryReplicatingObject query a replicating object information by object id