fix: refactor metadata config and change http address to grpc address

cmd/storage_provider/init.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/bnb-chain/greenfield-storage-provider/cmd/utils"
@@ -112,7 +111,7 @@ func initService(serviceName string, cfg *config.StorageProviderConfig) (server
 		if err != nil {
 			return nil, err
 		}
-		server, err = metadata.NewMetadataService(mCfg, context.Background())
+		server, err = metadata.NewMetadataService(mCfg)
 		if err != nil {
 			return nil, err
 		}

config/config.go

@@ -54,6 +54,7 @@ var DefaultStorageProviderConfig = &StorageProviderConfig{
 		model.StoneNodeService,
 		model.SyncerService,
 		model.SignerService,
+		model.MetadataService,
 	},
 	GRPCAddress: map[string]string{
 		model.UploaderService:   model.UploaderGRPCAddress,
@@ -62,10 +63,10 @@ var DefaultStorageProviderConfig = &StorageProviderConfig{
 		model.SyncerService:     model.SyncerGRPCAddress,
 		model.StoneNodeService:  model.StoneNodeGRPCAddress,
 		model.SignerService:     model.SignerGRPCAddress,
+		model.MetadataService:   model.MetaDataGRPCAddress,
 	},
 	HTTPAddress: map[string]string{
-		model.GatewayService:  model.GatewayHTTPAddress,
-		model.MetadataService: model.MetaDataServiceHTTPAddress,
+		model.GatewayService: model.GatewayHTTPAddress,
 	},
 	SpOperatorAddress: hex.EncodeToString([]byte("greenfield-storage-provider")),
 	Domain:            "gnfd.nodereal.com",

config/subconfig.go

@@ -51,6 +51,11 @@ func (cfg *StorageProviderConfig) MakeGatewayConfig() (*gateway.GatewayConfig, e
 	} else {
 		return nil, fmt.Errorf("missing syncer gPRC address configuration for gateway service")
 	}
+	if _, ok := cfg.GRPCAddress[model.MetadataService]; ok {
+		gCfg.MetadataServiceAddress = cfg.GRPCAddress[model.MetadataService]
+	} else {
+		return nil, fmt.Errorf("missing metadata gPRC address configuration for gateway service")
+	}
 	return gCfg, nil
 }
 
@@ -158,10 +163,10 @@ func (cfg *StorageProviderConfig) MakeMetadataServiceConfig() (*metadata.Metadat
 	mCfg := &metadata.MetadataConfig{
 		SpDBConfig: cfg.SpDBConfig,
 	}
-	if _, ok := cfg.HTTPAddress[model.MetadataService]; ok {
-		mCfg.Address = cfg.HTTPAddress[model.MetadataService]
+	if _, ok := cfg.GRPCAddress[model.MetadataService]; ok {
+		mCfg.GRPCAddress = cfg.GRPCAddress[model.MetadataService]
 	} else {
-		return nil, fmt.Errorf("missing meta data HTTP address configuration for mate data service")
+		return nil, fmt.Errorf("missing meta data gRPC address configuration for meta data service")
 	}
 	return mCfg, nil
 }

model/const.go

@@ -43,8 +43,6 @@ var SpServiceDesc = map[string]string{
 const (
 	// GatewayHTTPAddress default HTTP address of gateway
 	GatewayHTTPAddress = "localhost:9033"
-	// MetaDataServiceHTTPAddress default HTTP address of meta data service
-	MetaDataServiceHTTPAddress = "localhost:9733"
 	// UploaderGRPCAddress default gRPC address of uploader
 	UploaderGRPCAddress = "localhost:9133"
 	// DownloaderGRPCAddress default gRPC address of downloader
@@ -57,6 +55,8 @@ const (
 	SyncerGRPCAddress = "localhost:9533"
 	// SignerGRPCAddress default gRPC address of signer
 	SignerGRPCAddress = "localhost:9633"
+	// MetaDataGRPCAddress default gRPC address of meta data service
+	MetaDataGRPCAddress = "localhost:9733"
 )
 
 // define greenfield chain default address

service/gateway/gateway.go

@@ -86,10 +86,14 @@ func NewGatewayService(cfg *GatewayConfig) (*Gateway, error) {
 			return nil, err
 		}
 	}
-	if g.metadata, err = client.NewMetadataClient(g.config.MetadataServiceAddress); err != nil {
-		log.Warnw("failed to create metadata client", "err", err)
-		return nil, err
+
+	if cfg.MetadataServiceAddress != "" {
+		if g.metadata, err = client.NewMetadataClient(cfg.MetadataServiceAddress); err != nil {
+			log.Errorw("failed to create metadata client", "error", err)
+			return nil, err
+		}
 	}
+
 	log.Debugw("gateway succeed to init")
 	return g, nil
 }

service/gateway/metadata_handler.go

@@ -7,18 +7,38 @@ import (
 
 	"github.com/bnb-chain/greenfield-storage-provider/model"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
-	stypes "github.com/bnb-chain/greenfield-storage-provider/service/metadata/types"
-	"github.com/gorilla/mux"
+	metatypes "github.com/bnb-chain/greenfield-storage-provider/service/metadata/types"
 )
 
 // getUserBucketsHandler handle get object request
 func (g *Gateway) getUserBucketsHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
+	var (
+		err            error
+		errDescription *errorDescription
+		reqContext     *requestContext
+	)
 
-	req := &stypes.MetadataServiceGetUserBucketsRequest{
-		AccountId: vars["account_id"],
+	reqContext = newRequestContext(r)
+	defer func() {
+		if errDescription != nil {
+			_ = errDescription.errorJSONResponse(w, reqContext)
+		}
+		if errDescription != nil && errDescription.statusCode != http.StatusOK {
+			log.Errorf("action(%v) statusCode(%v) %v", getUserBucketsRouterName, errDescription.statusCode, reqContext.generateRequestDetail())
+		} else {
+			log.Infof("action(%v) statusCode(200) %v", getUserBucketsRouterName, reqContext.generateRequestDetail())
+		}
+	}()
+
+	if g.metadata == nil {
+		log.Errorw("failed to get user buckets due to not config metadata")
+		errDescription = NotExistComponentError
+		return
 	}
 
+	req := &metatypes.MetadataServiceGetUserBucketsRequest{
+		AccountId: reqContext.accountID,
+	}
 	ctx := log.Context(context.Background(), req)
 	resp, err := g.metadata.GetUserBuckets(ctx, req)
 	if err != nil {
@@ -36,11 +56,33 @@ func (g *Gateway) getUserBucketsHandler(w http.ResponseWriter, r *http.Request)
 
 // listObjectsByBucketNameHandler handle list objects by bucket name request
 func (g *Gateway) listObjectsByBucketNameHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
+	var (
+		err            error
+		errDescription *errorDescription
+		reqContext     *requestContext
+	)
+
+	reqContext = newRequestContext(r)
+	defer func() {
+		if errDescription != nil {
+			_ = errDescription.errorJSONResponse(w, reqContext)
+		}
+		if errDescription != nil && errDescription.statusCode != http.StatusOK {
+			log.Errorf("action(%v) statusCode(%v) %v", listObjectsByBucketRouterName, errDescription.statusCode, reqContext.generateRequestDetail())
+		} else {
+			log.Infof("action(%v) statusCode(200) %v", listObjectsByBucketRouterName, reqContext.generateRequestDetail())
+		}
+	}()
+
+	if g.metadata == nil {
+		log.Errorw("failed to list objects by bucket name due to not config metadata")
+		errDescription = NotExistComponentError
+		return
+	}
 
-	req := &stypes.MetadataServiceListObjectsByBucketNameRequest{
-		BucketName: vars["bucket_name"],
-		AccountId:  vars["account_id"],
+	req := &metatypes.MetadataServiceListObjectsByBucketNameRequest{
+		BucketName: reqContext.bucketName,
+		AccountId:  reqContext.accountID,
 	}
 
 	ctx := log.Context(context.Background(), req)

service/gateway/request_util.go

@@ -33,6 +33,8 @@ type requestContext struct {
 	skipAuth bool
 	// objectInfo is queried from the greenfield blockchain
 	objectInfo *storagetypes.ObjectInfo
+	// accountID is used to provide authentication to the sp
+	accountID string
 }
 
 // newRequestContext return a request context.
@@ -42,6 +44,7 @@ func newRequestContext(r *http.Request) *requestContext {
 		requestID:  util.GenerateRequestID(),
 		bucketName: vars["bucket"],
 		objectName: vars["object"],
+		accountID:  vars["account_id"],
 		request:    r,
 		startTime:  time.Now(),
 		vars:       vars,

service/gateway/response_util.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"encoding/json"
 	"encoding/xml"
 	"net/http"
 
@@ -66,6 +67,35 @@ func (desc *errorDescription) errorResponse(w http.ResponseWriter, reqCtx *reque
 	return nil
 }
 
+// errorJSONResponse is used to error response JSON.
+func (desc *errorDescription) errorJSONResponse(w http.ResponseWriter, reqCtx *requestContext) error {
+	var (
+		jsonBody []byte
+		err      error
+	)
+
+	var jsonInfo = struct {
+		Code      string `json:"Code"`
+		Message   string `json:"Message"`
+		RequestId string `json:"RequestId"`
+	}{
+		Code:      desc.errorCode,
+		Message:   desc.errorMessage,
+		RequestId: reqCtx.requestID,
+	}
+	if jsonBody, err = json.Marshal(&jsonInfo); err != nil {
+		return err
+	}
+
+	w.Header().Set(model.ContentTypeHeader, model.ContentTypeJSONHeaderValue)
+	w.WriteHeader(desc.statusCode)
+
+	if _, err = w.Write(jsonBody); err != nil {
+		return err
+	}
+	return nil
+}
+
 func generateContentRangeHeader(w http.ResponseWriter, start int64, end int64) {
 	if end < 0 {
 		w.Header().Set(model.ContentRangeHeader, "bytes "+util.Uint64ToString(uint64(start))+"-")

service/metadata/metadata_config.go

@@ -5,6 +5,6 @@ import (
 )
 
 type MetadataConfig struct {
-	Address    string
-	SpDBConfig *config.SQLDBConfig
+	GRPCAddress string
+	SpDBConfig  *config.SQLDBConfig
 }

service/metadata/service/metadata.go

@@ -33,6 +33,7 @@ func (metadata *Metadata) Name() string {
 // Start implement the lifecycle interface
 // to delete api/v1
 func (metadata *Metadata) Start(ctx context.Context) error {
+	metadata.ctx = ctx
 	if metadata.running.Swap(true) {
 		return errors.New("metadata has started")
 	}
@@ -45,7 +46,7 @@ func (metadata *Metadata) Start(ctx context.Context) error {
 
 // Serve starts grpc service.
 func (metadata *Metadata) serve(errCh chan error) {
-	lis, err := net.Listen("tcp", metadata.config.Address)
+	lis, err := net.Listen("tcp", metadata.config.GRPCAddress)
 	errCh <- err
 	if err != nil {
 		log.Errorw("failed to listen", "err", err)
@@ -71,12 +72,11 @@ func (metadata *Metadata) Stop(ctx context.Context) error {
 	return nil
 }
 
-func NewMetadataService(cfg *metadata.MetadataConfig, ctx context.Context) (metadata *Metadata, err error) {
+func NewMetadataService(cfg *metadata.MetadataConfig) (metadata *Metadata, err error) {
 	metadataStore, _ := store.NewStore(cfg.SpDBConfig)
 	metadata = &Metadata{
 		config: cfg,
 		name:   model.MetadataService,
-		ctx:    ctx,
 		store:  metadataStore,
 	}
 	return

service/metadata/store/store.go

@@ -8,11 +8,10 @@ import (
 	"github.com/bnb-chain/greenfield-storage-provider/store/config"
 	"github.com/bnb-chain/greenfield-storage-provider/store/sqldb"
 	_ "github.com/go-sql-driver/mysql"
-	"gorm.io/gorm"
 )
 
 type Store struct {
-	userDB *gorm.DB
+	spDB sqldb.SPDB
 }
 
 type IStore interface {
@@ -21,24 +20,13 @@ type IStore interface {
 }
 
 func NewStore(cfg *config.SQLDBConfig) (*Store, error) {
-	userDB, err := newGORM(cfg)
+	spDB, err := sqldb.NewSpDB(cfg)
 	if err != nil {
 		log.Errorf("fail to new gorm cfg:%v err:%v", cfg, err)
 		return nil, err
 	}
 
 	return &Store{
-		userDB: userDB,
+		spDB: spDB,
 	}, nil
 }
-
-func newGORM(cfg *config.SQLDBConfig) (*gorm.DB, error) {
-	db, err := sqldb.InitDB(cfg)
-
-	if err != nil {
-		log.Infof("fail to open database err:%v", err)
-		return nil, err
-	}
-
-	return db, nil
-}