feat: sp db adds service config interface (#164)

* feat: sp db adds service config interface

* fix: add table name

* fix: fix get db error

---------

Co-authored-by: DylanYong <dylan.y@nodereal.io>

Main refactor (#166)

* chore: add spdb address in env var

* fix: stone node replicate idx

* refactor: the entry and config

* fix: ci lint error

cmd/conf/conf.go

@@ -5,13 +5,12 @@ import (
 
 	"github.com/bnb-chain/greenfield-storage-provider/cmd/utils"
 	"github.com/bnb-chain/greenfield-storage-provider/config"
-	storeconf "github.com/bnb-chain/greenfield-storage-provider/store/config"
 )
 
 var ConfigDumpCmd = &cli.Command{
 	Action:   dumpConfigAction,
 	Name:     "config.dump",
-	Usage:    "Dump default configuration to file for editing",
+	Usage:    "Dump default configuration to the './config.toml' file for editing",
 	Category: "CONFIG COMMANDS",
 	Description: `
 The config.dump command writes default configuration 
@@ -32,6 +31,7 @@ var ConfigUploadCmd = &cli.Command{
 		utils.DBUserFlag,
 		utils.DBPasswordFlag,
 		utils.DBAddressFlag,
+		utils.DBDataBaseFlag,
 	},
 	Category: "CONFIG COMMANDS",
 	Description: `
@@ -42,13 +42,14 @@ SP_DB_USER and SP_DB_PASSWORD`,
 
 // configUploadAction is the config.upload command.
 func configUploadAction(ctx *cli.Context) error {
-	_ = config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name))
-	_ = &storeconf.SQLDBConfig{
-		User:     ctx.String(utils.DBUserFlag.Name),
-		Passwd:   ctx.String(utils.DBPasswordFlag.Name),
-		Address:  ctx.String(utils.DBAddressFlag.Name),
-		Database: "job_db",
+	spDB, err := utils.MakeSPDB(ctx)
+	if err != nil {
+		return err
 	}
-	// TODO:: new sp db and upload config
-	return nil
+	cfg := config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name))
+	cfgBytes, err := cfg.JsonMarshal()
+	if err != nil {
+		return err
+	}
+	return spDB.SetAllServiceConfigs("default", string(cfgBytes))
 }

cmd/storage_provider/init.go

@@ -4,85 +4,115 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/bnb-chain/greenfield-storage-provider/config"
-	"github.com/bnb-chain/greenfield-storage-provider/model"
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/cmd/utils"
 	"github.com/bnb-chain/greenfield-storage-provider/service/blocksyncer"
 	"github.com/bnb-chain/greenfield-storage-provider/service/challenge"
 	"github.com/bnb-chain/greenfield-storage-provider/service/downloader"
-	"github.com/bnb-chain/greenfield-storage-provider/service/gateway"
 	"github.com/bnb-chain/greenfield-storage-provider/service/metadata"
 	"github.com/bnb-chain/greenfield-storage-provider/service/signer"
 	"github.com/bnb-chain/greenfield-storage-provider/service/stonenode"
 	"github.com/bnb-chain/greenfield-storage-provider/service/syncer"
+	"github.com/urfave/cli/v2"
+
+	"github.com/bnb-chain/greenfield-storage-provider/config"
+	"github.com/bnb-chain/greenfield-storage-provider/model"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/service/gateway"
 	"github.com/bnb-chain/greenfield-storage-provider/service/uploader"
 )
 
+// initLog init global log level and log path.
+func initLog(ctx *cli.Context, cfg *config.StorageProviderConfig) error {
+	if cfg.LogCfg == nil {
+		cfg.LogCfg = config.DefaultLogConfig
+	}
+	if ctx.IsSet(utils.LogLevelFlag.Name) {
+		cfg.LogCfg.Level = ctx.String(utils.LogLevelFlag.Name)
+	}
+	if ctx.IsSet(utils.LogPathFlag.Name) {
+		cfg.LogCfg.Path = ctx.String(utils.LogPathFlag.Name)
+	}
+	if ctx.IsSet(utils.LogStdOutputFlag.Name) {
+		cfg.LogCfg.Path = ""
+	}
+	logLevel, err := log.ParseLevel(cfg.LogCfg.Level)
+	if err != nil {
+		return err
+	}
+	log.Init(logLevel, cfg.LogCfg.Path)
+	return nil
+}
+
 // initService init service instance by name and config.
 func initService(serviceName string, cfg *config.StorageProviderConfig) (server lifecycle.Service, err error) {
 	switch serviceName {
 	case model.GatewayService:
-		if cfg.GatewayCfg == nil {
-			cfg.GatewayCfg = config.DefaultStorageProviderConfig.GatewayCfg
+		gCfg, err := cfg.MakeGatewayConfig()
+		if err != nil {
+			return nil, err
 		}
-		server, err = gateway.NewGatewayService(cfg.GatewayCfg)
+		server, err = gateway.NewGatewayService(gCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.UploaderService:
-		if cfg.UploaderCfg == nil {
-			cfg.UploaderCfg = config.DefaultStorageProviderConfig.UploaderCfg
+		uCfg, err := cfg.MakeUploaderConfig()
+		if err != nil {
+			return nil, err
 		}
-		server, err = uploader.NewUploaderService(cfg.UploaderCfg)
+		server, err = uploader.NewUploaderService(uCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.DownloaderService:
-		if cfg.DownloaderCfg == nil {
-			cfg.DownloaderCfg = config.DefaultStorageProviderConfig.DownloaderCfg
-		}
-		server, err = downloader.NewDownloaderService(cfg.DownloaderCfg)
+		dCfg, err := cfg.MakeDownloaderConfig()
 		if err != nil {
 			return nil, err
 		}
-	case model.StoneNodeService:
-		if cfg.StoneNodeCfg == nil {
-			cfg.StoneNodeCfg = config.DefaultStorageProviderConfig.StoneNodeCfg
-		}
-		server, err = stonenode.NewStoneNodeService(cfg.StoneNodeCfg)
+		server, err = downloader.NewDownloaderService(dCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.SyncerService:
-		if cfg.SyncerCfg == nil {
-			cfg.SyncerCfg = config.DefaultStorageProviderConfig.SyncerCfg
+		sCfg, err := cfg.MakeSyncerConfig()
+		if err != nil {
+			return nil, err
 		}
-		server, err = syncer.NewSyncerService(cfg.SyncerCfg)
+		server, err = syncer.NewSyncerService(sCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.ChallengeService:
-		if cfg.ChallengeCfg == nil {
-			cfg.ChallengeCfg = config.DefaultStorageProviderConfig.ChallengeCfg
+		cCfg, err := cfg.MakeChallengeConfig()
+		if err != nil {
+			return nil, err
 		}
-		server, err = challenge.NewChallengeService(cfg.ChallengeCfg)
+		server, err = challenge.NewChallengeService(cCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.SignerService:
-		if cfg.SignerCfg == nil {
-			cfg.SignerCfg = config.DefaultStorageProviderConfig.SignerCfg
+		sCfg, _ := cfg.MakeSignerConfig()
+		server, err = signer.NewSignerServer(sCfg)
+		if err != nil {
+			return nil, err
 		}
-		server, err = signer.NewSignerServer(cfg.SignerCfg)
+	case model.StoneNodeService:
+		snCfg, err := cfg.MakeStoneNodeConfig()
+		if err != nil {
+			return nil, err
+		}
+		server, err = stonenode.NewStoneNodeService(snCfg)
 		if err != nil {
 			return nil, err
 		}
 	case model.MetadataService:
-		if cfg.MetadataCfg == nil {
-			cfg.MetadataCfg = config.DefaultStorageProviderConfig.MetadataCfg
+		mCfg, err := cfg.MakeMetadataServiceConfig()
+		if err != nil {
+			return nil, err
 		}
-		server, err = metadata.NewMetadataService(cfg.MetadataCfg, context.Background())
+		server, err = metadata.NewMetadataService(mCfg, context.Background())
 		if err != nil {
 			return nil, err
 		}

cmd/storage_provider/main.go

@@ -4,18 +4,16 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 	"syscall"
 
-	"github.com/bnb-chain/greenfield-storage-provider/model"
-	"github.com/bnb-chain/greenfield-storage-provider/util"
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
 	"github.com/urfave/cli/v2"
 
 	"github.com/bnb-chain/greenfield-storage-provider/cmd/conf"
 	"github.com/bnb-chain/greenfield-storage-provider/cmd/utils"
 	"github.com/bnb-chain/greenfield-storage-provider/config"
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/lifecycle"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
+	"github.com/bnb-chain/greenfield-storage-provider/util"
 )
 
 var (
@@ -28,15 +26,16 @@ var app *cli.App
 var (
 	// flags that configure the storage provider
 	spFlags = []cli.Flag{
-		utils.VersionFlag,
 		utils.ConfigFileFlag,
 		utils.ConfigRemoteFlag,
 		utils.ServerFlag,
 		utils.DBUserFlag,
 		utils.DBPasswordFlag,
 		utils.DBAddressFlag,
+		utils.DBDataBaseFlag,
 		utils.LogLevelFlag,
 		utils.LogPathFlag,
+		utils.LogStdOutputFlag,
 	}
 )
 
@@ -52,6 +51,7 @@ func init() {
 		conf.ConfigDumpCmd,
 		conf.ConfigUploadCmd,
 		// miscellaneous commands
+		VersionCmd,
 		utils.ListServiceCmd,
 	}
 }
@@ -65,66 +65,56 @@ func main() {
 
 // makeConfig loads the configuration and creates the storage provider backend.
 func makeConfig(ctx *cli.Context) (*config.StorageProviderConfig, error) {
-	cfg := &config.StorageProviderConfig{}
-	var services []string
-	if ctx.IsSet(utils.ServerFlag.Name) {
-		services = util.SplitByComma(ctx.String(utils.ServerFlag.Name))
-		for _, service := range services {
-			if _, ok := model.SpServiceDesc[strings.ToLower(service)]; !ok {
-				return nil, fmt.Errorf("invalid service %s", service)
-			}
-		}
-	}
+	// load config from remote db or local config file
+	cfg := config.DefaultStorageProviderConfig
 	if ctx.IsSet(utils.ConfigRemoteFlag.Name) {
-		//&storeconf.SQLDBConfig{
-		//	User:     ctx.String(utils.DBUserFlag.Name),
-		//	Passwd:   ctx.String(utils.DBPasswordFlag.Name),
-		//	Address:  ctx.String(utils.DBAddressFlag.Name),
-		//	Database: "job_db",
-		//}
-		// TODO:: new sp db load config by services
+		spDB, err := utils.MakeSPDB(ctx)
+		if err != nil {
+			return nil, err
+		}
+		_, cfgBytes, err := spDB.GetAllServiceConfigs()
+		if err != nil {
+			return nil, err
+		}
+		if err := cfg.JsonUnMarshal([]byte(cfgBytes)); err != nil {
+			return nil, err
+		}
 	} else if ctx.IsSet(utils.ConfigFileFlag.Name) {
 		cfg = config.LoadConfig(ctx.String(utils.ConfigFileFlag.Name))
-	} else {
-		cfg = config.DefaultStorageProviderConfig
 	}
+	// override the services to be started by flag
 	if ctx.IsSet(utils.ServerFlag.Name) {
+		services := util.SplitByComma(ctx.String(utils.ServerFlag.Name))
 		cfg.Service = services
 	}
+	// init log
+	if err := initLog(ctx, cfg); err != nil {
+		return nil, err
+	}
 	return cfg, nil
 }
 
 // storageProvider is the main entry point into the system if no special subcommand is ran.
 // It uses default config to run storage provider services based on the command line arguments
 // and runs it in blocking mode, waiting for it to be shut down.
 func storageProvider(ctx *cli.Context) error {
-	if ctx.IsSet(utils.VersionFlag.Name) {
-		fmt.Print(DumpLogo() + "\n" + DumpVersion())
-		return nil
-	}
-	logLevel, err := log.ParseLevel(ctx.String(utils.LogLevelFlag.Name))
-	if err != nil {
-		return err
-	}
-	log.Init(logLevel, ctx.String(utils.LogPathFlag.Name))
-
 	cfg, err := makeConfig(ctx)
 	if err != nil {
 		return err
 	}
 	slc := lifecycle.NewServiceLifecycle()
 	for _, serviceName := range cfg.Service {
-		// 1. init service instance.
+		// init service instance.
 		service, err := initService(serviceName, cfg)
 		if err != nil {
 			log.Errorw("failed to init service", "service", serviceName, "error", err)
 			os.Exit(1)
 		}
 		log.Debugw("success to init service ", "service", serviceName)
-		// 2. register service to lifecycle.
+		// register service to lifecycle.
 		slc.RegisterServices(service)
 	}
-	// 3. start all services and listen os signals.
+	// start all services and listen os signals.
 	slcCtx := context.Background()
 	slc.Signals(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP).StartServices(slcCtx).Wait(slcCtx)
 	return nil

cmd/storage_provider/version.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"runtime"
+
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -39,3 +41,19 @@ func DumpVersion() string {
 		CommitID,
 		runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildTime)
 }
+
+var VersionCmd = &cli.Command{
+	Action:   versionAction,
+	Name:     "version",
+	Aliases:  []string{"v"},
+	Usage:    "Print version information",
+	Category: "MISCELLANEOUS COMMANDS",
+	Description: `
+The output of this command is supposed to be machine-readable.
+`,
+}
+
+func versionAction(ctx *cli.Context) error {
+	fmt.Print(DumpLogo() + "\n" + DumpVersion())
+	return nil
+}

cmd/utils/db.go

@@ -0,0 +1,25 @@
+package utils
+
+import (
+	"github.com/bnb-chain/greenfield-storage-provider/config"
+	"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
+	"github.com/urfave/cli/v2"
+)
+
+// MakeSPDB return sp db instance
+func MakeSPDB(ctx *cli.Context) (*sqldb.SpDBImpl, error) {
+	spDBCfg := config.DefaultSQLDBConfig
+	if ctx.IsSet(ctx.String(DBUserFlag.Name)) {
+		spDBCfg.User = ctx.String(DBUserFlag.Name)
+	}
+	if ctx.IsSet(ctx.String(DBPasswordFlag.Name)) {
+		spDBCfg.Passwd = ctx.String(DBPasswordFlag.Name)
+	}
+	if ctx.IsSet(ctx.String(DBAddressFlag.Name)) {
+		spDBCfg.Address = ctx.String(DBAddressFlag.Name)
+	}
+	if ctx.IsSet(ctx.String(DBDataBaseFlag.Name)) {
+		spDBCfg.Database = ctx.String(DBDataBaseFlag.Name)
+	}
+	return sqldb.NewSpDB(spDBCfg)
+}