feat: update listobjects

deployment/localup/localup.sh

@@ -139,7 +139,7 @@ make_config() {
     sed -i -e "s/PProfHttpAddress = '.*'/PProfHttpAddress = '${pprof_address}'/g" config.toml
 
     # blocksyncer
-    sed -i -e "s/Modules = \[\]/Modules = \[\'epoch\',\'bucket\',\'object\',\'payment\',\'group\',\'permission\',\'storage_provider\'\]/g" config.toml
+    sed -i -e "s/Modules = \[\]/Modules = \[\'epoch\',\'bucket\',\'object\',\'payment\',\'group\',\'permission\',\'storage_provider\'\,\'prefix_tree\'\]/g" config.toml
     sed -i -e "s/RecreateTables = false/RecreateTables = true/g" config.toml
     WORKERS=50
     sed -i -e "s/Workers = 0/Workers = ${WORKERS}/g" config.toml

modular/blocksyncer/blocksyncer_options.go

@@ -17,7 +17,6 @@ import (
 	"github.com/forbole/juno/v4/models"
 	"github.com/forbole/juno/v4/modules"
 	"github.com/forbole/juno/v4/modules/messages"
-	"github.com/forbole/juno/v4/modules/registrar"
 	"github.com/forbole/juno/v4/node/remote"
 	"github.com/forbole/juno/v4/parser"
 	parserconfig "github.com/forbole/juno/v4/parser/config"
@@ -30,6 +29,8 @@ import (
 	coremodule "github.com/bnb-chain/greenfield-storage-provider/core/module"
 	"github.com/bnb-chain/greenfield-storage-provider/model"
 	"github.com/bnb-chain/greenfield-storage-provider/model/errors"
+	db "github.com/bnb-chain/greenfield-storage-provider/modular/blocksyncer/database"
+	registrar "github.com/bnb-chain/greenfield-storage-provider/modular/blocksyncer/modules"
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
 )
 
@@ -77,9 +78,9 @@ func (s *BlockSyncerModular) initClient() error {
 	// JunoConfig the runner
 	junoConfig := cmd.NewConfig("juno").
 		WithParseConfig(parsecmdtypes.NewConfig().
-			WithRegistrar(registrar.NewDefaultRegistrar(
+			WithRegistrar(registrar.NewBlockSyncerRegistrar(
 				messages.CosmosMessageAddressesParser,
-			)).WithFileType("toml"),
+			)).WithDBBuilder(db.BlockSyncerDBBuilder).WithFileType("toml"),
 		)
 	cmdCfg := junoConfig.GetParseConfig()
 	cmdCfg.WithTomlConfig(s.config)

modular/blocksyncer/database/database.go

@@ -0,0 +1,50 @@
+package database
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/forbole/juno/v4/database"
+	"github.com/forbole/juno/v4/database/mysql"
+	"github.com/forbole/juno/v4/database/sqlclient"
+	"gorm.io/gorm"
+)
+
+var _ database.Database = &DB{}
+
+// DB represents a SQL database with expanded features.
+// so that it can properly store custom BigDipper-related data.
+type DB struct {
+	*mysql.Database
+}
+
+// BlockSyncerDBBuilder allows to create a new DB instance implementing the db.Builder type
+func BlockSyncerDBBuilder(ctx *database.Context) (database.Database, error) {
+	db, err := sqlclient.New(&ctx.Cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &DB{
+		Database: &mysql.Database{
+			Impl: database.Impl{
+				Db:             db,
+				EncodingConfig: ctx.EncodingConfig,
+			},
+		},
+	}, nil
+}
+
+// Cast allows to cast the given db to a DB instance
+func Cast(db database.Database) *DB {
+	bdDatabase, ok := db.(*DB)
+	if !ok {
+		panic(fmt.Errorf("given database instance is not a DB"))
+	}
+	return bdDatabase
+}
+
+// errIsNotFound check if the error is not found
+func errIsNotFound(err error) bool {
+	return errors.Is(err, sql.ErrNoRows) || errors.Is(err, gorm.ErrRecordNotFound)
+}

modular/blocksyncer/database/masterdb.go

@@ -0,0 +1,28 @@
+package database
+
+import (
+	"context"
+
+	"github.com/forbole/juno/v4/models"
+	"gorm.io/gorm/clause"
+)
+
+// GetMasterDB get master db info
+func (db *DB) GetMasterDB(ctx context.Context) (*models.MasterDB, error) {
+	var masterDB models.MasterDB
+
+	err := db.Db.Find(&masterDB).Error
+	if err != nil && !errIsNotFound(err) {
+		return nil, err
+	}
+	return &masterDB, nil
+}
+
+// SetMasterDB set the master db
+func (db *DB) SetMasterDB(ctx context.Context, masterDB *models.MasterDB) error {
+	err := db.Db.Table((&models.MasterDB{}).TableName()).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "one_row_id"}},
+		DoUpdates: clause.AssignmentColumns([]string{"is_master"}),
+	}).Create(masterDB).Error
+	return err
+}

modular/blocksyncer/database/prefixtree.go

@@ -0,0 +1,78 @@
+package database
+
+import (
+	"context"
+
+	"github.com/bnb-chain/greenfield-storage-provider/store/bsdb"
+	"github.com/forbole/juno/v4/common"
+	"gorm.io/gorm"
+)
+
+// CreatePrefixTree create prefix tree nodes by input slice
+func (db *DB) CreatePrefixTree(ctx context.Context, prefixTree []*bsdb.SlashPrefixTreeNode) error {
+	err := db.Db.WithContext(ctx).Create(&prefixTree).Error
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeletePrefixTree delete prefix tree nodes by given conditions
+func (db *DB) DeletePrefixTree(ctx context.Context, prefixTree []*bsdb.SlashPrefixTreeNode) error {
+	if len(prefixTree) == 0 {
+		return nil
+	}
+	tx := db.Db.WithContext(ctx)
+	stmt := tx.Where("bucket_name = ? AND full_name = ? AND is_object = ?",
+		prefixTree[0].BucketName,
+		prefixTree[0].FullName,
+		prefixTree[0].IsObject)
+
+	for _, object := range prefixTree[1:] {
+		stmt = stmt.Or("bucket_name = ? AND full_name = ? AND is_object = ?",
+			object.BucketName,
+			object.FullName,
+			object.IsObject)
+	}
+	err := stmt.Unscoped().Delete(&bsdb.SlashPrefixTreeNode{}).Error
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetPrefixTree get prefix tree node by full name and bucket name
+func (db *DB) GetPrefixTree(ctx context.Context, fullName, bucketName string) (*bsdb.SlashPrefixTreeNode, error) {
+	var prefixTreeNode *bsdb.SlashPrefixTreeNode
+	err := db.Db.WithContext(ctx).Where("full_name = ? AND bucket_name = ? AND is_object = ?", fullName, bucketName, false).Take(&prefixTreeNode).Error
+	if err != nil {
+		if err == gorm.ErrRecordNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return prefixTreeNode, nil
+}
+
+// GetPrefixTreeObject get prefix tree node object by object id
+func (db *DB) GetPrefixTreeObject(ctx context.Context, objectID common.Hash) (*bsdb.SlashPrefixTreeNode, error) {
+	var prefixTreeNode *bsdb.SlashPrefixTreeNode
+	err := db.Db.WithContext(ctx).Where("object_id = ?", objectID).Take(&prefixTreeNode).Error
+	if err != nil {
+		if err == gorm.ErrRecordNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return prefixTreeNode, nil
+}
+
+// GetPrefixTreeCount get prefix tree nodes count by path and bucket name
+func (db *DB) GetPrefixTreeCount(ctx context.Context, pathName, bucketName string) (int64, error) {
+	var count int64
+	err := db.Db.WithContext(ctx).Table((&bsdb.SlashPrefixTreeNode{}).TableName()).Where("bucket_name = ? AND path_name = ?", bucketName, pathName).Count(&count).Error
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}

modular/blocksyncer/modules/prefixtree/module.go

@@ -0,0 +1,47 @@
+package prefixtree
+
+import (
+	"context"
+
+	"github.com/forbole/juno/v4/modules"
+	"gorm.io/gorm/schema"
+
+	"github.com/bnb-chain/greenfield-storage-provider/modular/blocksyncer/database"
+	"github.com/bnb-chain/greenfield-storage-provider/store/bsdb"
+)
+
+const (
+	ModuleName = "prefix_tree"
+)
+
+var (
+	_ modules.Module              = &Module{}
+	_ modules.PrepareTablesModule = &Module{}
+)
+
+// Module represents the object module
+type Module struct {
+	db *database.DB
+}
+
+// NewModule builds a new Module instance
+func NewModule(db *database.DB) *Module {
+	return &Module{
+		db: db,
+	}
+}
+
+// Name implements modules.Module
+func (m *Module) Name() string {
+	return ModuleName
+}
+
+// PrepareTables implements
+func (m *Module) PrepareTables() error {
+	return m.db.PrepareTables(context.TODO(), []schema.Tabler{&bsdb.SlashPrefixTreeNode{}})
+}
+
+// RecreateTables implements
+func (m *Module) RecreateTables() error {
+	return m.db.RecreateTables(context.TODO(), []schema.Tabler{&bsdb.SlashPrefixTreeNode{}})
+}