diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d5403b10..61173b4c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -21,7 +21,7 @@ jobs: # redis for tests services: redis: - image: redis + image: redis/redis-stack-server options: >- --health-cmd "redis-cli ping" --health-interval 10s diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf16b53a..b7740732 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: # redis for tests services: redis: - image: redis + image: redis/redis-stack-server options: >- --health-cmd "redis-cli ping" --health-interval 10s diff --git a/Makefile b/Makefile index 16757d80..eca4f31e 100644 --- a/Makefile +++ b/Makefile @@ -29,4 +29,4 @@ deps: go build -o deps github.com/gogo/protobuf/protoc-gen-gogofaster proto: - protoc --gogofaster_out=:. index/redisindex/indexproto/protos/*.proto + protoc --gogofaster_out=:. index/indexproto/protos/*.proto diff --git a/cmd/filenode.go b/cmd/filenode.go index 5f954dcd..d16a6d77 100644 --- a/cmd/filenode.go +++ b/cmd/filenode.go @@ -4,12 +4,13 @@ import ( "context" "flag" "fmt" - "github.com/anyproto/any-sync-filenode/account" - "github.com/anyproto/any-sync-filenode/config" - "github.com/anyproto/any-sync-filenode/filenode" - "github.com/anyproto/any-sync-filenode/index/redisindex" - "github.com/anyproto/any-sync-filenode/limit" - "github.com/anyproto/any-sync-filenode/redisprovider" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "syscall" + "time" + "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/coordinator/coordinatorclient" @@ -24,12 +25,13 @@ import ( "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/nodeconf/nodeconfstore" "go.uber.org/zap" - "net/http" - _ "net/http/pprof" - "os" - "os/signal" - "syscall" - "time" + + "github.com/anyproto/any-sync-filenode/account" + "github.com/anyproto/any-sync-filenode/config" + "github.com/anyproto/any-sync-filenode/filenode" + "github.com/anyproto/any-sync-filenode/index" + "github.com/anyproto/any-sync-filenode/limit" + "github.com/anyproto/any-sync-filenode/redisprovider" // import this to keep govvv in go.mod on mod tidy _ "github.com/ahmetb/govvv/integration-test/app-different-package/mypkg" @@ -114,7 +116,7 @@ func Bootstrap(a *app.App) { Register(limit.New()). Register(store()). Register(redisprovider.New()). - Register(redisindex.New()). + Register(index.New()). Register(metric.New()). Register(server.New()). Register(filenode.New()) diff --git a/cmd/filenode_test.go b/cmd/filenode_test.go index 6dd447ef..55db994d 100644 --- a/cmd/filenode_test.go +++ b/cmd/filenode_test.go @@ -2,15 +2,17 @@ package main import ( "context" - "github.com/anyproto/any-sync-filenode/config" - "github.com/anyproto/any-sync-filenode/store/s3store" + "os" + "testing" + commonaccount "github.com/anyproto/any-sync/accountservice" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/metric" "github.com/anyproto/any-sync/net/rpc" "github.com/stretchr/testify/require" - "os" - "testing" + + "github.com/anyproto/any-sync-filenode/config" + "github.com/anyproto/any-sync-filenode/store/s3store" ) var ctx = context.Background() @@ -27,7 +29,8 @@ func TestBootstrap(t *testing.T) { Drpc: rpc.Config{}, Metric: metric.Config{}, S3Store: s3store.Config{ - Bucket: "test", + Bucket: "test", + IndexBucket: "testIndex", }, FileDevStore: config.FileDevStore{}, NetworkStorePath: tmpDir, diff --git a/etc/any-sync-filenode.yml b/etc/any-sync-filenode.yml index 2d42ff57..babba9c6 100755 --- a/etc/any-sync-filenode.yml +++ b/etc/any-sync-filenode.yml @@ -21,6 +21,7 @@ s3Store: region: eu-central-1 profile: default bucket: anytype-test + indexBucket: anytype-test maxThreads: 16 redis: diff --git a/filenode/filenode.go b/filenode/filenode.go index 6e857b2c..31222e48 100644 --- a/filenode/filenode.go +++ b/filenode/filenode.go @@ -3,11 +3,7 @@ package filenode import ( "context" "errors" - "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/index/redisindex" - "github.com/anyproto/any-sync-filenode/limit" - "github.com/anyproto/any-sync-filenode/store" - "github.com/anyproto/any-sync-filenode/testutil" + "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/commonfile/fileblockstore" @@ -17,6 +13,11 @@ import ( "github.com/anyproto/any-sync/net/rpc/server" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-filenode/index" + "github.com/anyproto/any-sync-filenode/limit" + "github.com/anyproto/any-sync-filenode/store" ) const CName = "filenode.filenode" @@ -45,7 +46,7 @@ type fileNode struct { func (fn *fileNode) Init(a *app.App) (err error) { fn.store = a.MustComponent(fileblockstore.CName).(store.Store) - fn.index = a.MustComponent(redisindex.CName).(index.Index) + fn.index = a.MustComponent(index.CName).(index.Index) fn.limit = a.MustComponent(limit.CName).(limit.Limit) fn.handler = &rpcHandler{f: fn} fn.metric = a.MustComponent(metric.CName).(metric.Metric) @@ -57,7 +58,7 @@ func (fn *fileNode) Name() (name string) { } func (fn *fileNode) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { - exists, err := fn.index.Exists(ctx, k) + exists, err := fn.index.CidExists(ctx, k) if err != nil { return nil, err } @@ -72,12 +73,12 @@ func (fn *fileNode) Add(ctx context.Context, spaceId string, fileId string, bs [ if err != nil { return err } - unlock, err := fn.index.Lock(ctx, testutil.BlocksToKeys(bs)) + unlock, err := fn.index.BlocksLock(ctx, bs) if err != nil { return err } defer unlock() - toUpload, err := fn.index.GetNonExistentBlocks(ctx, bs) + toUpload, err := fn.index.BlocksGetNonExistent(ctx, bs) if err != nil { return err } @@ -85,12 +86,20 @@ func (fn *fileNode) Add(ctx context.Context, spaceId string, fileId string, bs [ if err = fn.store.Add(ctx, toUpload); err != nil { return err } + if err = fn.index.BlocksAdd(ctx, bs); err != nil { + return err + } } - return fn.index.Bind(ctx, storeKey, fileId, bs) + cidEntries, err := fn.index.CidEntriesByBlocks(ctx, bs) + if err != nil { + return err + } + defer cidEntries.Release() + return fn.index.FileBind(ctx, storeKey, fileId, cidEntries) } func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid) (result []*fileproto.BlockAvailability, err error) { - var storeKey string + var storeKey index.Key if spaceId != "" { if storeKey, err = fn.StoreKey(ctx, spaceId, false); err != nil { return @@ -99,7 +108,7 @@ func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid) result = make([]*fileproto.BlockAvailability, 0, len(cids)) var inSpaceM = make(map[string]struct{}) if spaceId != "" { - inSpace, err := fn.index.ExistsInStorage(ctx, storeKey, cids) + inSpace, err := fn.index.CidExistsInSpace(ctx, storeKey, cids) if err != nil { return nil, err } @@ -116,7 +125,7 @@ func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid) res.Status = fileproto.AvailabilityStatus_ExistsInSpace } else { var ex bool - if ex, err = fn.index.Exists(ctx, k); err != nil { + if ex, err = fn.index.CidExists(ctx, k); err != nil { return nil, err } else if ex { res.Status = fileproto.AvailabilityStatus_Exists @@ -132,60 +141,101 @@ func (fn *fileNode) BlocksBind(ctx context.Context, spaceId, fileId string, cids if err != nil { return err } - unlock, err := fn.index.Lock(ctx, cids) + cidEntries, err := fn.index.CidEntries(ctx, cids) if err != nil { return err } - defer unlock() - return fn.index.BindCids(ctx, storeKey, fileId, cids) + defer cidEntries.Release() + return fn.index.FileBind(ctx, storeKey, fileId, cidEntries) } -func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit bool) (storageKey string, err error) { +func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit bool) (storageKey index.Key, err error) { if spaceId == "" { - return "", fileprotoerr.ErrForbidden + return storageKey, fileprotoerr.ErrForbidden } // this call also confirms that space exists and valid - limitBytes, storageKey, err := fn.limit.Check(ctx, spaceId) + limitBytes, groupId, err := fn.limit.Check(ctx, spaceId) if err != nil { return } - if storageKey != spaceId { - // try to move store to the new key - mErr := fn.index.MoveStorage(ctx, spaceId, storageKey) - if mErr != nil && mErr != index.ErrStorageNotFound && mErr != index.ErrTargetStorageExists { - return "", mErr - } + storageKey = index.Key{ + GroupId: groupId, + SpaceId: spaceId, + } + + if e := fn.index.Migrate(ctx, storageKey); e != nil { + log.WarnCtx(ctx, "space migrate error", zap.String("spaceId", spaceId), zap.Error(e)) } if checkLimit { - currentSize, e := fn.index.StorageSize(ctx, storageKey) + info, e := fn.index.GroupInfo(ctx, groupId) if e != nil { - return "", e + return storageKey, e } - if currentSize >= limitBytes { - return "", fileprotoerr.ErrSpaceLimitExceeded + if info.BytesUsage >= limitBytes { + return storageKey, fileprotoerr.ErrSpaceLimitExceeded } } return } func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *fileproto.SpaceInfoResponse, err error) { - info = &fileproto.SpaceInfoResponse{} + var ( + storageKey = index.Key{SpaceId: spaceId} + limitBytes uint64 + ) + if limitBytes, storageKey.GroupId, err = fn.limit.Check(ctx, spaceId); err != nil { + return nil, err + } + groupInfo, err := fn.index.GroupInfo(ctx, storageKey.GroupId) + if err != nil { + return nil, err + } + if info, err = fn.spaceInfo(ctx, storageKey, groupInfo); err != nil { + return nil, err + } + info.LimitBytes = limitBytes + return +} + +func (fn *fileNode) AccountInfo(ctx context.Context) (info *fileproto.AccountInfoResponse, err error) { + info = &fileproto.AccountInfoResponse{} // we have space/identity validation in limit.Check - var storageKey string - if info.LimitBytes, storageKey, err = fn.limit.Check(ctx, spaceId); err != nil { + var groupId string + + if info.LimitBytes, groupId, err = fn.limit.Check(ctx, ""); err != nil { return nil, err } - if info.UsageBytes, err = fn.index.StorageSize(ctx, storageKey); err != nil { + + groupInfo, err := fn.index.GroupInfo(ctx, groupId) + if err != nil { return nil, err } - si, err := fn.index.StorageInfo(ctx, storageKey) + info.TotalCidsCount = groupInfo.CidsCount + info.TotalUsageBytes = groupInfo.BytesUsage + for _, spaceId := range groupInfo.SpaceIds { + spaceInfo, err := fn.spaceInfo(ctx, index.Key{GroupId: groupId, SpaceId: spaceId}, groupInfo) + if err != nil { + return nil, err + } + spaceInfo.LimitBytes = info.LimitBytes + info.Spaces = append(info.Spaces, spaceInfo) + } + return +} + +func (fn *fileNode) spaceInfo(ctx context.Context, key index.Key, groupInfo index.GroupInfo) (info *fileproto.SpaceInfoResponse, err error) { + info = &fileproto.SpaceInfoResponse{} + info.SpaceId = key.SpaceId + spaceInfo, err := fn.index.SpaceInfo(ctx, key) if err != nil { return nil, err } - info.CidsCount = uint64(si.CidCount) - info.FilesCount = uint64(si.FileCount) + info.TotalUsageBytes = groupInfo.BytesUsage + info.FilesCount = uint64(spaceInfo.FileCount) + info.CidsCount = spaceInfo.CidsCount + info.SpaceUsageBytes = spaceInfo.BytesUsage return } @@ -194,26 +244,25 @@ func (fn *fileNode) FilesDelete(ctx context.Context, spaceId string, fileIds []s if err != nil { return } - for _, fileId := range fileIds { - if err = fn.index.UnBind(ctx, storeKey, fileId); err != nil { - return - } - } - return + return fn.index.FileUnbind(ctx, storeKey, fileIds...) } -func (fn *fileNode) FileInfo(ctx context.Context, spaceId, fileId string) (info *fileproto.FileInfo, err error) { +func (fn *fileNode) FileInfo(ctx context.Context, spaceId string, fileIds ...string) (info []*fileproto.FileInfo, err error) { storeKey, err := fn.StoreKey(ctx, spaceId, false) if err != nil { return } - fi, err := fn.index.FileInfo(ctx, storeKey, fileId) + fis, err := fn.index.FileInfo(ctx, storeKey, fileIds...) if err != nil { return nil, err } - return &fileproto.FileInfo{ - FileId: fileId, - UsageBytes: fi.BytesUsage, - CidsCount: fi.CidCount, - }, nil + info = make([]*fileproto.FileInfo, len(fis)) + for i, fi := range fis { + info[i] = &fileproto.FileInfo{ + FileId: fileIds[i], + UsageBytes: fi.BytesUsage, + CidsCount: uint32(fi.CidsCount), + } + } + return } diff --git a/filenode/filenode_test.go b/filenode/filenode_test.go index d0ef4a83..e03e34c3 100644 --- a/filenode/filenode_test.go +++ b/filenode/filenode_test.go @@ -2,14 +2,8 @@ package filenode import ( "context" - "github.com/anyproto/any-sync-filenode/config" - "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/index/mock_index" - "github.com/anyproto/any-sync-filenode/index/redisindex" - "github.com/anyproto/any-sync-filenode/limit" - "github.com/anyproto/any-sync-filenode/limit/mock_limit" - "github.com/anyproto/any-sync-filenode/store/mock_store" - "github.com/anyproto/any-sync-filenode/testutil" + "testing" + "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/commonfile/fileblockstore" "github.com/anyproto/any-sync/commonfile/fileproto" @@ -21,7 +15,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "testing" + + "github.com/anyproto/any-sync-filenode/config" + "github.com/anyproto/any-sync-filenode/index" + "github.com/anyproto/any-sync-filenode/index/mock_index" + "github.com/anyproto/any-sync-filenode/limit" + "github.com/anyproto/any-sync-filenode/limit/mock_limit" + "github.com/anyproto/any-sync-filenode/store/mock_store" + "github.com/anyproto/any-sync-filenode/testutil" ) var ctx = context.Background() @@ -31,22 +32,23 @@ func TestFileNode_Add(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - storeKey = "sk:" + spaceId + storeKey = newRandKey() fileId = testutil.NewRandCid().String() b = testutil.NewRandBlock(1024) ) - fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) - fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() - fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(120), nil) - fx.index.EXPECT().Lock(ctx, []cid.Cid{b.Cid()}).Return(func() {}, nil) - fx.index.EXPECT().GetNonExistentBlocks(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil) + fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: uint64(120)}, nil) + fx.index.EXPECT().BlocksLock(ctx, []blocks.Block{b}).Return(func() {}, nil) + fx.index.EXPECT().BlocksGetNonExistent(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil) fx.store.EXPECT().Add(ctx, []blocks.Block{b}) - fx.index.EXPECT().Bind(ctx, storeKey, fileId, []blocks.Block{b}) + fx.index.EXPECT().BlocksAdd(ctx, []blocks.Block{b}) + fx.index.EXPECT().CidEntriesByBlocks(ctx, []blocks.Block{b}).Return(&index.CidEntries{}, nil) + fx.index.EXPECT().FileBind(ctx, storeKey, fileId, gomock.Any()) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, FileId: fileId, Cid: b.Cid().Bytes(), Data: b.RawData(), @@ -54,22 +56,22 @@ func TestFileNode_Add(t *testing.T) { require.NoError(t, err) require.NotNil(t, resp) }) + t.Run("limit exceed", func(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - storeKey = "sk:" + spaceId + storeKey = newRandKey() fileId = testutil.NewRandCid().String() b = testutil.NewRandBlock(1024) ) - fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) - fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() - fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(124), nil) + fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: uint64(124)}, nil) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, FileId: fileId, Cid: b.Cid().Bytes(), Data: b.RawData(), @@ -81,14 +83,14 @@ func TestFileNode_Add(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - fileId = testutil.NewRandCid().String() - b = testutil.NewRandBlock(1024) - b2 = testutil.NewRandBlock(10) + storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + b = testutil.NewRandBlock(1024) + b2 = testutil.NewRandBlock(10) ) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, FileId: fileId, Cid: b2.Cid().Bytes(), Data: b.RawData(), @@ -114,6 +116,7 @@ func TestFileNode_Add(t *testing.T) { require.EqualError(t, err, fileprotoerr.ErrQuerySizeExceeded.Error()) assert.Nil(t, resp) }) + } func TestFileNode_Get(t *testing.T) { @@ -122,7 +125,7 @@ func TestFileNode_Get(t *testing.T) { defer fx.Finish(t) spaceId := testutil.NewRandSpaceId() b := testutil.NewRandBlock(10) - fx.index.EXPECT().Exists(gomock.Any(), b.Cid()).Return(true, nil) + fx.index.EXPECT().CidExists(gomock.Any(), b.Cid()).Return(true, nil) fx.store.EXPECT().Get(ctx, b.Cid()).Return(b, nil) resp, err := fx.handler.BlockGet(ctx, &fileproto.BlockGetRequest{ SpaceId: spaceId, @@ -136,7 +139,7 @@ func TestFileNode_Get(t *testing.T) { defer fx.Finish(t) spaceId := testutil.NewRandSpaceId() b := testutil.NewRandBlock(10) - fx.index.EXPECT().Exists(gomock.Any(), b.Cid()).Return(false, nil) + fx.index.EXPECT().CidExists(gomock.Any(), b.Cid()).Return(false, nil) resp, err := fx.handler.BlockGet(ctx, &fileproto.BlockGetRequest{ SpaceId: spaceId, Cid: b.Cid().Bytes(), @@ -149,20 +152,19 @@ func TestFileNode_Get(t *testing.T) { func TestFileNode_Check(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) - var spaceId = testutil.NewRandSpaceId() - var storeKey = "sk:" + spaceId + var storeKey = newRandKey() var bs = testutil.NewRandBlocks(3) cids := make([][]byte, len(bs)) for _, b := range bs { cids = append(cids, b.Cid().Bytes()) } - fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(100000), storeKey, nil) - fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() - fx.index.EXPECT().ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil) - fx.index.EXPECT().Exists(ctx, bs[1].Cid()).Return(true, nil) - fx.index.EXPECT().Exists(ctx, bs[2].Cid()).Return(false, nil) + fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(100000), storeKey.GroupId, nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().CidExistsInSpace(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil) + fx.index.EXPECT().CidExists(ctx, bs[1].Cid()).Return(true, nil) + fx.index.EXPECT().CidExists(ctx, bs[2].Cid()).Return(false, nil) resp, err := fx.handler.BlocksCheck(ctx, &fileproto.BlocksCheckRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, Cids: cids, }) require.NoError(t, err) @@ -176,31 +178,32 @@ func TestFileNode_BlocksBind(t *testing.T) { fx := newFixture(t) defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - storeKey = "sk:" + spaceId - fileId = testutil.NewRandCid().String() - bs = testutil.NewRandBlocks(3) - cidsB = make([][]byte, len(bs)) - cids = make([]cid.Cid, len(bs)) + storeKey = newRandKey() + fileId = testutil.NewRandCid().String() + bs = testutil.NewRandBlocks(3) + cidsB = make([][]byte, len(bs)) + cids = make([]cid.Cid, len(bs)) + cidEntries = &index.CidEntries{} ) for i, b := range bs { cids[i] = b.Cid() cidsB[i] = b.Cid().Bytes() } - fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) - fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() - fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(12), nil) - fx.index.EXPECT().Lock(ctx, cids).Return(func() {}, nil) - fx.index.EXPECT().BindCids(ctx, storeKey, fileId, cids) + fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).Return(uint64(123), storeKey.GroupId, nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{BytesUsage: 12}, nil) + fx.index.EXPECT().CidEntries(ctx, cids).Return(cidEntries, nil) + fx.index.EXPECT().FileBind(ctx, storeKey, fileId, cidEntries) resp, err := fx.handler.BlocksBind(ctx, &fileproto.BlocksBindRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, FileId: fileId, Cids: cidsB, }) require.NotNil(t, resp) require.NoError(t, err) + } func TestFileNode_FileInfo(t *testing.T) { @@ -208,18 +211,16 @@ func TestFileNode_FileInfo(t *testing.T) { defer fx.Finish(t) var ( - spaceId = testutil.NewRandSpaceId() - storeKey = "sk:" + spaceId + storeKey = newRandKey() fileId1 = testutil.NewRandCid().String() fileId2 = testutil.NewRandCid().String() ) - fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes().Return(uint64(100000), storeKey, nil) - fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() - fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1).Return(index.FileInfo{1, 1}, nil) - fx.index.EXPECT().FileInfo(ctx, storeKey, fileId2).Return(index.FileInfo{2, 2}, nil) + fx.limit.EXPECT().Check(ctx, storeKey.SpaceId).AnyTimes().Return(uint64(100000), storeKey.GroupId, nil) + fx.index.EXPECT().Migrate(ctx, storeKey) + fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1, fileId2).Return([]index.FileInfo{{1, 1}, {2, 2}}, nil) resp, err := fx.handler.FilesInfo(ctx, &fileproto.FilesInfoRequest{ - SpaceId: spaceId, + SpaceId: storeKey.SpaceId, FileIds: []string{fileId1, fileId2}, }) require.NoError(t, err) @@ -228,6 +229,39 @@ func TestFileNode_FileInfo(t *testing.T) { assert.Equal(t, uint64(2), resp.FilesInfo[1].UsageBytes) } +func TestFileNode_AccountInfo(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + var ( + storeKey = newRandKey() + ) + fx.limit.EXPECT().Check(ctx, "").AnyTimes().Return(uint64(100000), storeKey.GroupId, nil) + fx.index.EXPECT().GroupInfo(ctx, storeKey.GroupId).Return(index.GroupInfo{ + BytesUsage: 100, + CidsCount: 10, + SpaceIds: []string{storeKey.SpaceId}, + }, nil) + fx.index.EXPECT().SpaceInfo(ctx, storeKey).Return(index.SpaceInfo{ + BytesUsage: 90, + CidsCount: 9, + FileCount: 1, + }, nil) + + resp, err := fx.handler.AccountInfo(ctx, &fileproto.AccountInfoRequest{}) + require.NoError(t, err) + require.Len(t, resp.Spaces, 1) + assert.Equal(t, uint64(100), resp.TotalUsageBytes) + assert.Equal(t, uint64(10), resp.TotalCidsCount) + assert.Equal(t, uint64(100000), resp.LimitBytes) + + assert.Equal(t, uint64(90), resp.Spaces[0].SpaceUsageBytes) + assert.Equal(t, uint64(9), resp.Spaces[0].CidsCount) + assert.Equal(t, uint64(1), resp.Spaces[0].FilesCount) + assert.Equal(t, uint64(100000), resp.Spaces[0].LimitBytes) + assert.Equal(t, uint64(100), resp.Spaces[0].TotalUsageBytes) +} + func newFixture(t *testing.T) *fixture { ctrl := gomock.NewController(t) fx := &fixture{ @@ -240,8 +274,10 @@ func newFixture(t *testing.T) *fixture { a: new(app.App), } - fx.index.EXPECT().Name().Return(redisindex.CName).AnyTimes() + fx.index.EXPECT().Name().Return(index.CName).AnyTimes() fx.index.EXPECT().Init(gomock.Any()).AnyTimes() + fx.index.EXPECT().Run(gomock.Any()).AnyTimes() + fx.index.EXPECT().Close(gomock.Any()).AnyTimes() fx.store.EXPECT().Name().Return(fileblockstore.CName).AnyTimes() fx.store.EXPECT().Init(gomock.Any()).AnyTimes() @@ -270,3 +306,10 @@ func (fx *fixture) Finish(t *testing.T) { fx.ctrl.Finish() require.NoError(t, fx.a.Close(ctx)) } + +func newRandKey() index.Key { + return index.Key{ + SpaceId: testutil.NewRandSpaceId(), + GroupId: "A" + testutil.NewRandCid().String(), + } +} diff --git a/filenode/rpchandler.go b/filenode/rpchandler.go index 5ed4e633..761faada 100644 --- a/filenode/rpchandler.go +++ b/filenode/rpchandler.go @@ -2,18 +2,19 @@ package filenode import ( "context" + "time" + "github.com/anyproto/any-sync/commonfile/fileproto" "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" "github.com/anyproto/any-sync/metric" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "go.uber.org/zap" - "time" ) const ( cidSizeLimit = 2 << 20 // 2 Mb - fileInfoReqLimit = 100 + fileInfoReqLimit = 1000 ) type rpcHandler struct { @@ -163,21 +164,11 @@ func (r rpcHandler) FilesInfo(ctx context.Context, req *fileproto.FilesInfoReque err = fileprotoerr.ErrQuerySizeExceeded return } - resp = &fileproto.FilesInfoResponse{ - FilesInfo: make([]*fileproto.FileInfo, len(req.FileIds)), - } - _, err = r.f.StoreKey(ctx, req.SpaceId, false) + resp = &fileproto.FilesInfoResponse{} + resp.FilesInfo, err = r.f.FileInfo(ctx, req.SpaceId, req.FileIds...) if err != nil { return nil, err } - var info *fileproto.FileInfo - for i, fileId := range req.FileIds { - info, err = r.f.FileInfo(ctx, req.SpaceId, fileId) - if err != nil { - return nil, err - } - resp.FilesInfo[i] = info - } return resp, nil } @@ -211,6 +202,21 @@ func (r rpcHandler) SpaceInfo(ctx context.Context, req *fileproto.SpaceInfoReque return } +func (r rpcHandler) AccountInfo(ctx context.Context, req *fileproto.AccountInfoRequest) (resp *fileproto.AccountInfoResponse, err error) { + st := time.Now() + defer func() { + r.f.metric.RequestLog(ctx, + "file.accountInfo", + metric.TotalDur(time.Since(st)), + zap.Error(err), + ) + }() + if resp, err = r.f.AccountInfo(ctx); err != nil { + return + } + return +} + func convertCids(bCids [][]byte) (cids []cid.Cid) { cids = make([]cid.Cid, 0, len(bCids)) var uniqMap map[string]struct{} diff --git a/go.mod b/go.mod index 8641e450..8778ae75 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/anyproto/any-sync-filenode go 1.21 require ( + github.com/OneOfOne/xxhash v1.2.2 github.com/ahmetb/govvv v0.3.0 - github.com/anyproto/any-sync v0.3.1 + github.com/anyproto/any-sync v0.3.5 github.com/aws/aws-sdk-go v1.46.3 + github.com/cespare/xxhash/v2 v2.2.0 github.com/go-redsync/redsync/v4 v4.10.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 @@ -26,16 +28,14 @@ require ( github.com/anyproto/go-slip21 v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect @@ -60,21 +60,21 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect - github.com/quic-go/qtls-go1-20 v0.3.3 // indirect - github.com/quic-go/quic-go v0.38.1 // indirect + github.com/quic-go/qtls-go1-20 v0.3.4 // indirect + github.com/quic-go/quic-go v0.39.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/zeebo/errs v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.13.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect google.golang.org/protobuf v1.31.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 834ae9ff..fd74e456 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0= github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s= -github.com/anyproto/any-sync v0.3.1 h1:AHsIYyhM9J+eqKVjnsuGgT/4u+f47JyEhfIBMnDLIIA= -github.com/anyproto/any-sync v0.3.1/go.mod h1:v0w3l3FBWjzNgg5t8aWlI+aYkcA8kLaoJFfr/GHsWYk= +github.com/anyproto/any-sync v0.3.5 h1:G7hqF2ww8oNs1C3Tve/k3B9PucaFU0UOkS4iopLYA1I= +github.com/anyproto/any-sync v0.3.5/go.mod h1:b5VBU4kw1df4ezTYZd9iPEVzPZcDfTGfjis6gUortQc= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i6IA= @@ -65,8 +65,8 @@ github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs0 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -145,18 +145,18 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= -github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= -github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= +github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= +github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.39.1 h1:d/m3oaN/SD2c+f7/yEjZxe2zEVotXprnrCCJ2y/ZZFE= +github.com/quic-go/quic-go v0.39.1/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= @@ -204,8 +204,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -224,8 +224,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -248,8 +248,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -264,7 +264,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= diff --git a/index/bind.go b/index/bind.go new file mode 100644 index 00000000..bf1cfb44 --- /dev/null +++ b/index/bind.go @@ -0,0 +1,121 @@ +package index + +import ( + "context" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +func (ri *redisIndex) FileBind(ctx context.Context, key Key, fileId string, cids *CidEntries) (err error) { + var ( + sk = spaceKey(key) + gk = groupKey(key) + ) + _, gRelease, err := ri.AcquireKey(ctx, gk) + if err != nil { + return + } + defer gRelease() + _, sRelease, err := ri.AcquireKey(ctx, sk) + if err != nil { + return + } + defer sRelease() + + // get file entry + fileInfo, isNewFile, err := ri.getFileEntry(ctx, key, fileId) + if err != nil { + return + } + + // make a list of indexes of non-exists cids + var newFileCidIdx = make([]int, 0, len(cids.entries)) + for i, c := range cids.entries { + if !fileInfo.Exists(c.Cid.String()) { + newFileCidIdx = append(newFileCidIdx, i) + fileInfo.Cids = append(fileInfo.Cids, c.Cid.String()) + fileInfo.Size_ += c.Size_ + } + } + + // all cids exists, nothing to do + if len(newFileCidIdx) == 0 { + return + } + + // get all cids from space and group in one pipeline + var ( + cidExistSpaceCmds = make([]*redis.BoolCmd, len(newFileCidIdx)) + cidExistGroupCmds = make([]*redis.BoolCmd, len(newFileCidIdx)) + ) + _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { + for i, idx := range newFileCidIdx { + ck := cidKey(cids.entries[idx].Cid) + cidExistSpaceCmds[i] = pipe.HExists(ctx, sk, ck) + cidExistGroupCmds[i] = pipe.HExists(ctx, gk, ck) + } + return nil + }) + if err != nil { + return + } + + // load group and space info + spaceInfo, err := ri.getSpaceEntry(ctx, key) + if err != nil { + return + } + groupInfo, err := ri.getGroupEntry(ctx, key) + if err != nil { + return + } + + // calculate new group and space stats + for i, idx := range newFileCidIdx { + ex, err := cidExistGroupCmds[i].Result() + if err != nil { + return err + } + if !ex { + spaceInfo.CidCount++ + spaceInfo.Size_ += cids.entries[idx].Size_ + } + ex, err = cidExistSpaceCmds[i].Result() + if err != nil { + return err + } + if !ex { + groupInfo.CidCount++ + groupInfo.Size_ += cids.entries[idx].Size_ + } + } + groupInfo.AddSpaceId(key.SpaceId) + if isNewFile { + spaceInfo.FileCount++ + } + + // make group and space updates in one tx + _, err = ri.cl.TxPipelined(ctx, func(tx redis.Pipeliner) error { + // increment cid refs + for _, idx := range newFileCidIdx { + ck := cidKey(cids.entries[idx].Cid) + tx.HIncrBy(ctx, gk, ck, 1) + tx.HIncrBy(ctx, sk, ck, 1) + } + // save info + spaceInfo.Save(ctx, key, tx) + groupInfo.Save(ctx, key, tx) + fileInfo.Save(ctx, key, fileId, tx) + return nil + }) + + // update cids + for _, idx := range newFileCidIdx { + cids.entries[idx].Refs++ + if saveErr := cids.entries[idx].Save(ctx, ri.cl); saveErr != nil { + log.WarnCtx(ctx, "unable to save cid info", zap.Error(saveErr), zap.String("cid", cids.entries[idx].Cid.String())) + } + } + return +} diff --git a/index/bind_test.go b/index/bind_test.go new file mode 100644 index 00000000..5d4fae35 --- /dev/null +++ b/index/bind_test.go @@ -0,0 +1,100 @@ +package index + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_Bind(t *testing.T) { + t.Run("first add", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + key := newRandKey() + fileId := testutil.NewRandCid().String() + require.NoError(t, fx.BlocksAdd(ctx, bs)) + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer cids.Release() + + require.NoError(t, fx.FileBind(ctx, key, fileId, cids)) + fInfo, err := fx.FileInfo(ctx, key, fileId) + require.NoError(t, err) + assert.Equal(t, uint64(len(bs)), fInfo[0].CidsCount) + assert.Equal(t, sumSize, fInfo[0].BytesUsage) + }) + + t.Run("two files with same cids", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + key := newRandKey() + fileId1 := testutil.NewRandCid().String() + fileId2 := testutil.NewRandCid().String() + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + cidsA, err := fx.CidEntriesByBlocks(ctx, bs[:2]) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, key, fileId1, cidsA)) + cidsA.Release() + + cidsB, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer cidsB.Release() + require.NoError(t, fx.FileBind(ctx, key, fileId2, cidsB)) + + spaceInfo, err := fx.SpaceInfo(ctx, key) + require.NoError(t, err) + assert.Equal(t, SpaceInfo{ + BytesUsage: sumSize, + FileCount: 2, + CidsCount: uint64(len(bs)), + }, spaceInfo) + + groupInfo, err := fx.GroupInfo(ctx, key.GroupId) + require.NoError(t, err) + assert.Equal(t, GroupInfo{ + BytesUsage: sumSize, + CidsCount: uint64(len(bs)), + SpaceIds: []string{key.SpaceId}, + }, groupInfo) + + }) + + t.Run("bind twice", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + bs := testutil.NewRandBlocks(3) + var sumSize uint64 + for _, b := range bs { + sumSize += uint64(len(b.RawData())) + } + key := newRandKey() + fileId := testutil.NewRandCid().String() + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer cids.Release() + + require.NoError(t, fx.FileBind(ctx, key, fileId, cids)) + require.NoError(t, fx.FileBind(ctx, key, fileId, cids)) + + fInfo, err := fx.FileInfo(ctx, key, fileId) + require.NoError(t, err) + assert.Equal(t, uint64(len(bs)), fInfo[0].CidsCount) + assert.Equal(t, sumSize, fInfo[0].BytesUsage) + }) +} diff --git a/index/cidentry.go b/index/cidentry.go new file mode 100644 index 00000000..f2eac8ce --- /dev/null +++ b/index/cidentry.go @@ -0,0 +1,39 @@ +package index + +import ( + "context" + "time" + + "github.com/ipfs/go-cid" + "github.com/redis/go-redis/v9" + + "github.com/anyproto/any-sync-filenode/index/indexproto" +) + +type CidEntries struct { + entries []*cidEntry +} + +func (ce *CidEntries) Release() { + for _, entry := range ce.entries { + if entry.release != nil { + entry.release() + } + } + return +} + +type cidEntry struct { + Cid cid.Cid + release func() + *indexproto.CidEntry +} + +func (ce *cidEntry) Save(ctx context.Context, cl redis.Cmdable) error { + ce.UpdateTime = time.Now().Unix() + data, err := ce.Marshal() + if err != nil { + return err + } + return cl.Set(ctx, cidKey(ce.Cid), data, 0).Err() +} diff --git a/index/cids.go b/index/cids.go new file mode 100644 index 00000000..275972d4 --- /dev/null +++ b/index/cids.go @@ -0,0 +1,189 @@ +package index + +import ( + "context" + "errors" + "time" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/redis/go-redis/v9" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-filenode/index/indexproto" +) + +const ( + cidCount = "cidCount.{system}" + cidSizeSumKey = "cidSizeSum.{system}" +) + +func (ri *redisIndex) CidExists(ctx context.Context, c cid.Cid) (ok bool, err error) { + return ri.CheckKey(ctx, cidKey(c)) +} + +func (ri *redisIndex) CidEntries(ctx context.Context, cids []cid.Cid) (entries *CidEntries, err error) { + entries = &CidEntries{} + for _, c := range cids { + if err = ri.getAndAddToEntries(ctx, entries, c); err != nil { + return nil, err + } + } + return entries, nil +} + +func (ri *redisIndex) CidEntriesByString(ctx context.Context, cids []string) (entries *CidEntries, err error) { + entries = &CidEntries{} + var c cid.Cid + for _, cs := range cids { + c, err = cid.Decode(cs) + if err != nil { + return + } + if err = ri.getAndAddToEntries(ctx, entries, c); err != nil { + return nil, err + } + } + return entries, nil +} + +func (ri *redisIndex) CidEntriesByBlocks(ctx context.Context, bs []blocks.Block) (entries *CidEntries, err error) { + entries = &CidEntries{} + var visited = make(map[string]struct{}) + for _, b := range bs { + if _, ok := visited[b.Cid().KeyString()]; ok { + continue + } + if err = ri.getAndAddToEntries(ctx, entries, b.Cid()); err != nil { + return nil, err + } + visited[b.Cid().KeyString()] = struct{}{} + } + return entries, nil +} + +func (ri *redisIndex) getAndAddToEntries(ctx context.Context, entries *CidEntries, c cid.Cid) (err error) { + ok, release, err := ri.AcquireKey(ctx, cidKey(c)) + if err != nil { + return + } + if !ok { + release() + return ErrCidsNotExist + } + entry, err := ri.getCidEntry(ctx, c) + if err != nil { + release() + return err + } + entry.release = release + entries.entries = append(entries.entries, entry) + return +} + +func (ri *redisIndex) BlocksAdd(ctx context.Context, bs []blocks.Block) (err error) { + for _, b := range bs { + exists, release, err := ri.AcquireKey(ctx, cidKey(b.Cid())) + if err != nil { + return err + } + if !exists { + if _, err = ri.createCidEntry(ctx, b); err != nil { + release() + return err + } + } else { + log.WarnCtx(ctx, "attempt to add existing block", zap.String("cid", b.Cid().String())) + } + release() + } + return +} + +func (ri *redisIndex) CidExistsInSpace(ctx context.Context, k Key, cids []cid.Cid) (exists []cid.Cid, err error) { + _, release, err := ri.AcquireKey(ctx, spaceKey(k)) + if err != nil { + return + } + defer release() + var existsRes = make([]*redis.BoolCmd, len(cids)) + _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { + for i, c := range cids { + existsRes[i] = pipe.HExists(ctx, spaceKey(k), cidKey(c)) + } + return nil + }) + for i, c := range cids { + ex, e := existsRes[i].Result() + if e != nil { + return nil, e + } + if ex { + exists = append(exists, c) + } + } + return +} + +func (ri *redisIndex) getCidEntry(ctx context.Context, c cid.Cid) (entry *cidEntry, err error) { + ck := cidKey(c) + cidData, err := ri.cl.Get(ctx, ck).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + err = ErrCidsNotExist + } + return + } + protoEntry := &indexproto.CidEntry{} + err = protoEntry.Unmarshal([]byte(cidData)) + if err != nil { + return + } + entry = &cidEntry{ + Cid: c, + CidEntry: protoEntry, + } + if err = ri.initCidEntry(ctx, entry); err != nil { + return nil, err + } + return +} + +func (ri *redisIndex) createCidEntry(ctx context.Context, b blocks.Block) (entry *cidEntry, err error) { + now := time.Now().Unix() + entry = &cidEntry{ + Cid: b.Cid(), + CidEntry: &indexproto.CidEntry{ + Size_: uint64(len(b.RawData())), + CreateTime: now, + UpdateTime: now, + }, + } + if err = ri.initCidEntry(ctx, entry); err != nil { + return nil, err + } + return +} + +func (ri *redisIndex) initCidEntry(ctx context.Context, entry *cidEntry) (err error) { + if entry.Version == 0 { + entry.Version = 1 + if err != nil { + return + } + if err = entry.Save(ctx, ri.cl); err != nil { + return + } + _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { + if e := pipe.IncrBy(ctx, cidSizeSumKey, int64(entry.Size_)).Err(); e != nil { + return e + } + if e := pipe.Incr(ctx, cidCount).Err(); e != nil { + return e + } + return nil + }) + return err + } + return +} diff --git a/index/cids_test.go b/index/cids_test.go new file mode 100644 index 00000000..1610b20b --- /dev/null +++ b/index/cids_test.go @@ -0,0 +1,130 @@ +package index + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync-filenode/index/indexproto" + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_BlocksAdd(t *testing.T) { + bs := testutil.NewRandBlocks(5) + fx := newFixture(t) + defer fx.Finish(t) + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + result, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer result.Release() + + require.Len(t, result.entries, len(bs)) + for _, e := range result.entries { + assert.NotEmpty(t, e.Size_) + assert.NotEmpty(t, e.CreateTime) + assert.NotEmpty(t, e.UpdateTime) + assert.NotEmpty(t, e.Version) + } +} + +func TestRedisIndex_CidEntries(t *testing.T) { + t.Run("success", func(t *testing.T) { + bs := testutil.NewRandBlocks(5) + fx := newFixture(t) + defer fx.Finish(t) + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + cids := testutil.BlocksToKeys(bs) + + result, err := fx.CidEntries(ctx, cids) + defer result.Release() + require.NoError(t, err) + require.Len(t, result.entries, len(bs)) + }) + t.Run("not all cids", func(t *testing.T) { + bs := testutil.NewRandBlocks(5) + fx := newFixture(t) + defer fx.Finish(t) + + require.NoError(t, fx.BlocksAdd(ctx, bs[:3])) + + cids := testutil.BlocksToKeys(bs) + + _, err := fx.CidEntries(ctx, cids) + assert.EqualError(t, err, ErrCidsNotExist.Error()) + }) + t.Run("migrate old cids", func(t *testing.T) { + bs := testutil.NewRandBlocks(5) + fx := newFixture(t) + defer fx.Finish(t) + + for _, b := range bs { + // save old entry, without version + entry := &cidEntry{ + Cid: b.Cid(), + CidEntry: &indexproto.CidEntry{ + Size_: uint64(len(b.RawData())), + CreateTime: 1, + UpdateTime: 2, + }, + } + require.NoError(t, entry.Save(ctx, fx.cl)) + } + + cids := testutil.BlocksToKeys(bs) + + result, err := fx.CidEntries(ctx, cids) + defer result.Release() + require.NoError(t, err) + require.Len(t, result.entries, len(bs)) + for _, e := range result.entries { + assert.NotEmpty(t, e.Size_) + assert.NotEmpty(t, e.CreateTime) + assert.NotEmpty(t, e.UpdateTime) + assert.NotEmpty(t, e.Version) + } + }) +} + +func TestRedisIndex_CidExistsInSpace(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + key := newRandKey() + + bs := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + + cids, err := fx.CidEntriesByBlocks(ctx, bs[:2]) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, key, "fileId", cids)) + cids.Release() + + exists, err := fx.CidExistsInSpace(ctx, key, testutil.BlocksToKeys(bs)) + require.NoError(t, err) + + require.Len(t, exists, 2) + assert.Equal(t, testutil.BlocksToKeys(bs[:2]), exists) +} + +func TestRedisIndex_CidExists(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + bs := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs[:2])) + + for i, b := range bs { + ok, err := fx.CidExists(ctx, b.Cid()) + require.NoError(t, err) + if i < 2 { + assert.True(t, ok) + } else { + assert.False(t, ok) + } + } +} diff --git a/index/entry.go b/index/entry.go new file mode 100644 index 00000000..b962d3af --- /dev/null +++ b/index/entry.go @@ -0,0 +1,123 @@ +package index + +import ( + "context" + "errors" + "slices" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/anyproto/any-sync-filenode/index/indexproto" +) + +type fileEntry struct { + *indexproto.FileEntry +} + +func (f *fileEntry) Exists(c string) (ok bool) { + return slices.Contains(f.Cids, c) +} + +func (f *fileEntry) Save(ctx context.Context, k Key, fileId string, cl redis.Pipeliner) { + f.UpdateTime = time.Now().Unix() + data, err := f.Marshal() + if err != nil { + return + } + cl.HSet(ctx, spaceKey(k), fileKey(fileId), data) +} + +func (ri *redisIndex) getFileEntry(ctx context.Context, k Key, fileId string) (entry *fileEntry, isCreated bool, err error) { + result, err := ri.cl.HGet(ctx, spaceKey(k), fileKey(fileId)).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return + } + if errors.Is(err, redis.Nil) { + return &fileEntry{ + FileEntry: &indexproto.FileEntry{ + CreateTime: time.Now().Unix(), + }, + }, true, nil + } + fileEntryProto := &indexproto.FileEntry{} + if err = fileEntryProto.Unmarshal([]byte(result)); err != nil { + return + } + return &fileEntry{FileEntry: fileEntryProto}, false, nil +} + +type spaceEntry struct { + *indexproto.SpaceEntry +} + +func (f *spaceEntry) Save(ctx context.Context, k Key, cl redis.Pipeliner) { + f.UpdateTime = time.Now().Unix() + data, err := f.Marshal() + if err != nil { + return + } + cl.HSet(ctx, spaceKey(k), infoKey, data) +} + +func (ri *redisIndex) getSpaceEntry(ctx context.Context, key Key) (entry *spaceEntry, err error) { + result, err := ri.cl.HGet(ctx, spaceKey(key), infoKey).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return + } + if errors.Is(err, redis.Nil) { + now := time.Now().Unix() + return &spaceEntry{ + SpaceEntry: &indexproto.SpaceEntry{ + CreateTime: now, + UpdateTime: now, + GroupId: key.GroupId, + }, + }, nil + } + spaceEntryProto := &indexproto.SpaceEntry{} + if err = spaceEntryProto.Unmarshal([]byte(result)); err != nil { + return + } + return &spaceEntry{SpaceEntry: spaceEntryProto}, nil +} + +type groupEntry struct { + *indexproto.GroupEntry +} + +func (f *groupEntry) Save(ctx context.Context, k Key, cl redis.Pipeliner) { + f.UpdateTime = time.Now().Unix() + data, err := f.Marshal() + if err != nil { + return + } + cl.HSet(ctx, groupKey(k), infoKey, data) +} + +func (f *groupEntry) AddSpaceId(spaceId string) { + if !slices.Contains(f.SpaceIds, spaceId) { + f.SpaceIds = append(f.SpaceIds, spaceId) + } +} + +func (ri *redisIndex) getGroupEntry(ctx context.Context, key Key) (entry *groupEntry, err error) { + result, err := ri.cl.HGet(ctx, groupKey(key), infoKey).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return + } + if errors.Is(err, redis.Nil) { + now := time.Now().Unix() + return &groupEntry{ + GroupEntry: &indexproto.GroupEntry{ + CreateTime: now, + UpdateTime: now, + }, + }, nil + } + groupEntryProto := &indexproto.GroupEntry{} + if err = groupEntryProto.Unmarshal([]byte(result)); err != nil { + return + } + return &groupEntry{GroupEntry: groupEntryProto}, nil +} diff --git a/index/index.go b/index/index.go index 147376ae..c4899cfd 100644 --- a/index/index.go +++ b/index/index.go @@ -4,41 +4,236 @@ package index import ( "context" "errors" + "strconv" + "time" + + "github.com/OneOfOne/xxhash" "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/util/periodicsync" + "github.com/go-redsync/redsync/v4" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/redis/go-redis/v9" + + "github.com/anyproto/any-sync-filenode/redisprovider" + "github.com/anyproto/any-sync-filenode/store/s3store" ) +const CName = "filenode.index" + +var log = logger.NewNamed(CName) + var ( - ErrCidsNotExist = errors.New("cids not exist") - ErrTargetStorageExists = errors.New("target storage exists") - ErrStorageNotFound = errors.New("storage not found") + ErrCidsNotExist = errors.New("cids not exist") ) type Index interface { - Exists(ctx context.Context, k cid.Cid) (exists bool, err error) - IsAllExists(ctx context.Context, cids []cid.Cid) (exists bool, err error) - StorageInfo(ctx context.Context, key string) (info StorageInfo, err error) - GetNonExistentBlocks(ctx context.Context, bs []blocks.Block) (nonExists []blocks.Block, err error) - Bind(ctx context.Context, key, fileId string, bs []blocks.Block) error - BindCids(ctx context.Context, key string, fileId string, cids []cid.Cid) error - UnBind(ctx context.Context, key, fileId string) (err error) - ExistsInStorage(ctx context.Context, key string, ks []cid.Cid) (exists []cid.Cid, err error) - FileInfo(ctx context.Context, key, fileId string) (info FileInfo, err error) - StorageSize(ctx context.Context, key string) (size uint64, err error) - Lock(ctx context.Context, ks []cid.Cid) (unlock func(), err error) - AddBlocks(ctx context.Context, upload []blocks.Block) error - MoveStorage(ctx context.Context, fromKey, toKey string) error - app.Component -} - -type StorageInfo struct { - Key string - FileCount int - CidCount int + FileBind(ctx context.Context, key Key, fileId string, cidEntries *CidEntries) (err error) + FileUnbind(ctx context.Context, kye Key, fileIds ...string) (err error) + FileInfo(ctx context.Context, key Key, fileIds ...string) (fileInfo []FileInfo, err error) + + GroupInfo(ctx context.Context, groupId string) (info GroupInfo, err error) + SpaceInfo(ctx context.Context, key Key) (info SpaceInfo, err error) + + BlocksGetNonExistent(ctx context.Context, bs []blocks.Block) (nonExistent []blocks.Block, err error) + BlocksLock(ctx context.Context, bs []blocks.Block) (unlock func(), err error) + BlocksAdd(ctx context.Context, bs []blocks.Block) (err error) + + CidExists(ctx context.Context, c cid.Cid) (ok bool, err error) + CidEntries(ctx context.Context, cids []cid.Cid) (entries *CidEntries, err error) + CidEntriesByBlocks(ctx context.Context, bs []blocks.Block) (entries *CidEntries, err error) + CidExistsInSpace(ctx context.Context, k Key, cids []cid.Cid) (exists []cid.Cid, err error) + + Migrate(ctx context.Context, key Key) error + + app.ComponentRunnable +} + +func New() Index { + return &redisIndex{} +} + +type Key struct { + GroupId string + SpaceId string +} + +type GroupInfo struct { + BytesUsage uint64 + CidsCount uint64 + SpaceIds []string +} + +type SpaceInfo struct { + BytesUsage uint64 + CidsCount uint64 + FileCount uint32 } type FileInfo struct { BytesUsage uint64 - CidCount uint32 + CidsCount uint64 +} + +/* + Redis db structure: + CIDS: + c:{cid}: proto(Entry) + cidCount.{system}: int + cidSizeSum.{system}: int + STORES: + g:{groupId}: map + c:{cidId} -> int(refCount) + info: proto(GroupEntry} + s:{spaceId}: map + f:{fileId}: proto(FileEntry) + c:{cidId} -> int(refCount) + info: proto(SpaceEntry) + +*/ + +type redisIndex struct { + cl redis.UniversalClient + redsync *redsync.Redsync + persistStore persistentStore + persistTtl time.Duration + ticker periodicsync.PeriodicSync +} + +func (ri *redisIndex) Init(a *app.App) (err error) { + ri.cl = a.MustComponent(redisprovider.CName).(redisprovider.RedisProvider).Redis() + ri.persistStore = a.MustComponent(s3store.CName).(persistentStore) + ri.redsync = redsync.New(goredis.NewPool(ri.cl)) + // todo: move to config + ri.persistTtl = time.Hour + return +} + +func (ri *redisIndex) Name() (name string) { + return CName +} + +func (ri *redisIndex) Run(ctx context.Context) (err error) { + ri.ticker = periodicsync.NewPeriodicSync(60, time.Minute*10, func(ctx context.Context) error { + ri.PersistKeys(ctx) + return nil + }, log) + ri.ticker.Run() + return +} + +func (ri *redisIndex) FileInfo(ctx context.Context, key Key, fileIds ...string) (fileInfos []FileInfo, err error) { + _, release, err := ri.AcquireKey(ctx, spaceKey(key)) + if err != nil { + return + } + defer release() + fileInfos = make([]FileInfo, len(fileIds)) + for i, fileId := range fileIds { + fEntry, _, err := ri.getFileEntry(ctx, key, fileId) + if err != nil { + return nil, err + } + fileInfos[i] = FileInfo{ + BytesUsage: fEntry.Size_, + CidsCount: uint64(len(fEntry.Cids)), + } + } + return +} + +func (ri *redisIndex) BlocksGetNonExistent(ctx context.Context, bs []blocks.Block) (nonExistent []blocks.Block, err error) { + for _, b := range bs { + ex, err := ri.CheckKey(ctx, cidKey(b.Cid())) + if err != nil { + return nil, err + } + if !ex { + nonExistent = append(nonExistent, b) + } + } + return +} + +func (ri *redisIndex) BlocksLock(ctx context.Context, bs []blocks.Block) (unlock func(), err error) { + var lockers = make([]*redsync.Mutex, 0, len(bs)) + + unlock = func() { + for _, l := range lockers { + _, _ = l.Unlock() + } + } + + for _, b := range bs { + l := ri.redsync.NewMutex("_lock:b:"+b.Cid().String(), redsync.WithExpiry(time.Minute)) + if err = l.LockContext(ctx); err != nil { + unlock() + return nil, err + } + lockers = append(lockers, l) + } + return +} + +func (ri *redisIndex) GroupInfo(ctx context.Context, groupId string) (info GroupInfo, err error) { + _, release, err := ri.AcquireKey(ctx, groupKey(Key{GroupId: groupId})) + if err != nil { + return + } + defer release() + sEntry, err := ri.getGroupEntry(ctx, Key{GroupId: groupId}) + if err != nil { + return + } + return GroupInfo{ + BytesUsage: sEntry.Size_, + CidsCount: sEntry.CidCount, + SpaceIds: sEntry.SpaceIds, + }, nil } + +func (ri *redisIndex) SpaceInfo(ctx context.Context, key Key) (info SpaceInfo, err error) { + _, release, err := ri.AcquireKey(ctx, spaceKey(key)) + if err != nil { + return + } + defer release() + sEntry, err := ri.getSpaceEntry(ctx, key) + if err != nil { + return + } + return SpaceInfo{ + BytesUsage: sEntry.Size_, + CidsCount: sEntry.CidCount, + FileCount: sEntry.FileCount, + }, nil +} + +func (ri *redisIndex) Close(ctx context.Context) error { + if ri.ticker != nil { + ri.ticker.Close() + } + return nil +} + +func cidKey(c cid.Cid) string { + return "c:" + c.String() +} + +func spaceKey(k Key) string { + hash := strconv.FormatUint(uint64(xxhash.ChecksumString32(k.GroupId)), 36) + return "s:" + k.SpaceId + ".{" + hash + "}" +} + +func groupKey(k Key) string { + hash := strconv.FormatUint(uint64(xxhash.ChecksumString32(k.GroupId)), 36) + return "g:" + k.GroupId + ".{" + hash + "}" +} + +func fileKey(fileId string) string { + return "f:" + fileId +} + +const infoKey = "info" diff --git a/index/index_test.go b/index/index_test.go new file mode 100644 index 00000000..a8f8b5e8 --- /dev/null +++ b/index/index_test.go @@ -0,0 +1,52 @@ +package index + +import ( + "context" + "testing" + + "github.com/anyproto/any-sync/app" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-filenode/redisprovider/testredisprovider" + "github.com/anyproto/any-sync-filenode/store/mock_store" + "github.com/anyproto/any-sync-filenode/store/s3store" + "github.com/anyproto/any-sync-filenode/testutil" +) + +var ctx = context.Background() + +func newRandKey() Key { + return Key{ + SpaceId: testutil.NewRandSpaceId(), + GroupId: "A" + testutil.NewRandCid().String(), + } +} + +func newFixture(t *testing.T) (fx *fixture) { + ctrl := gomock.NewController(t) + fx = &fixture{ + redisIndex: New().(*redisIndex), + ctrl: ctrl, + persistStore: mock_store.NewMockStore(ctrl), + a: new(app.App), + } + fx.persistStore.EXPECT().Name().Return(s3store.CName).AnyTimes() + fx.persistStore.EXPECT().Init(gomock.Any()).AnyTimes() + + fx.a.Register(testredisprovider.NewTestRedisProvider()).Register(fx.redisIndex).Register(fx.persistStore) + require.NoError(t, fx.a.Start(ctx)) + return +} + +type fixture struct { + *redisIndex + a *app.App + ctrl *gomock.Controller + persistStore *mock_store.MockStore +} + +func (fx *fixture) Finish(t require.TestingT) { + require.NoError(t, fx.a.Close(ctx)) + fx.ctrl.Finish() +} diff --git a/index/indexproto/index.pb.go b/index/indexproto/index.pb.go new file mode 100644 index 00000000..80851f09 --- /dev/null +++ b/index/indexproto/index.pb.go @@ -0,0 +1,1615 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: index/indexproto/protos/index.proto + +package indexproto + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type CidEntry struct { + Size_ uint64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` + CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` + UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` + Refs int32 `protobuf:"varint,4,opt,name=refs,proto3" json:"refs,omitempty"` + Version uint32 `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"` +} + +func (m *CidEntry) Reset() { *m = CidEntry{} } +func (m *CidEntry) String() string { return proto.CompactTextString(m) } +func (*CidEntry) ProtoMessage() {} +func (*CidEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_f1f29953df8d243b, []int{0} +} +func (m *CidEntry) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CidEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CidEntry.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CidEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_CidEntry.Merge(m, src) +} +func (m *CidEntry) XXX_Size() int { + return m.Size() +} +func (m *CidEntry) XXX_DiscardUnknown() { + xxx_messageInfo_CidEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_CidEntry proto.InternalMessageInfo + +func (m *CidEntry) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *CidEntry) GetCreateTime() int64 { + if m != nil { + return m.CreateTime + } + return 0 +} + +func (m *CidEntry) GetUpdateTime() int64 { + if m != nil { + return m.UpdateTime + } + return 0 +} + +func (m *CidEntry) GetRefs() int32 { + if m != nil { + return m.Refs + } + return 0 +} + +func (m *CidEntry) GetVersion() uint32 { + if m != nil { + return m.Version + } + return 0 +} + +type CidList struct { + Cids [][]byte `protobuf:"bytes,1,rep,name=cids,proto3" json:"cids,omitempty"` +} + +func (m *CidList) Reset() { *m = CidList{} } +func (m *CidList) String() string { return proto.CompactTextString(m) } +func (*CidList) ProtoMessage() {} +func (*CidList) Descriptor() ([]byte, []int) { + return fileDescriptor_f1f29953df8d243b, []int{1} +} +func (m *CidList) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CidList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CidList.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CidList) XXX_Merge(src proto.Message) { + xxx_messageInfo_CidList.Merge(m, src) +} +func (m *CidList) XXX_Size() int { + return m.Size() +} +func (m *CidList) XXX_DiscardUnknown() { + xxx_messageInfo_CidList.DiscardUnknown(m) +} + +var xxx_messageInfo_CidList proto.InternalMessageInfo + +func (m *CidList) GetCids() [][]byte { + if m != nil { + return m.Cids + } + return nil +} + +type GroupEntry struct { + GroupId string `protobuf:"bytes,1,opt,name=groupId,proto3" json:"groupId,omitempty"` + CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` + UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` + Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + CidCount uint64 `protobuf:"varint,5,opt,name=cidCount,proto3" json:"cidCount,omitempty"` + SpaceIds []string `protobuf:"bytes,6,rep,name=spaceIds,proto3" json:"spaceIds,omitempty"` +} + +func (m *GroupEntry) Reset() { *m = GroupEntry{} } +func (m *GroupEntry) String() string { return proto.CompactTextString(m) } +func (*GroupEntry) ProtoMessage() {} +func (*GroupEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_f1f29953df8d243b, []int{2} +} +func (m *GroupEntry) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GroupEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GroupEntry.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GroupEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupEntry.Merge(m, src) +} +func (m *GroupEntry) XXX_Size() int { + return m.Size() +} +func (m *GroupEntry) XXX_DiscardUnknown() { + xxx_messageInfo_GroupEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupEntry proto.InternalMessageInfo + +func (m *GroupEntry) GetGroupId() string { + if m != nil { + return m.GroupId + } + return "" +} + +func (m *GroupEntry) GetCreateTime() int64 { + if m != nil { + return m.CreateTime + } + return 0 +} + +func (m *GroupEntry) GetUpdateTime() int64 { + if m != nil { + return m.UpdateTime + } + return 0 +} + +func (m *GroupEntry) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *GroupEntry) GetCidCount() uint64 { + if m != nil { + return m.CidCount + } + return 0 +} + +func (m *GroupEntry) GetSpaceIds() []string { + if m != nil { + return m.SpaceIds + } + return nil +} + +type SpaceEntry struct { + GroupId string `protobuf:"bytes,1,opt,name=groupId,proto3" json:"groupId,omitempty"` + CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` + UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` + Size_ uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + FileCount uint32 `protobuf:"varint,5,opt,name=fileCount,proto3" json:"fileCount,omitempty"` + CidCount uint64 `protobuf:"varint,6,opt,name=cidCount,proto3" json:"cidCount,omitempty"` +} + +func (m *SpaceEntry) Reset() { *m = SpaceEntry{} } +func (m *SpaceEntry) String() string { return proto.CompactTextString(m) } +func (*SpaceEntry) ProtoMessage() {} +func (*SpaceEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_f1f29953df8d243b, []int{3} +} +func (m *SpaceEntry) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SpaceEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SpaceEntry.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SpaceEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_SpaceEntry.Merge(m, src) +} +func (m *SpaceEntry) XXX_Size() int { + return m.Size() +} +func (m *SpaceEntry) XXX_DiscardUnknown() { + xxx_messageInfo_SpaceEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_SpaceEntry proto.InternalMessageInfo + +func (m *SpaceEntry) GetGroupId() string { + if m != nil { + return m.GroupId + } + return "" +} + +func (m *SpaceEntry) GetCreateTime() int64 { + if m != nil { + return m.CreateTime + } + return 0 +} + +func (m *SpaceEntry) GetUpdateTime() int64 { + if m != nil { + return m.UpdateTime + } + return 0 +} + +func (m *SpaceEntry) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *SpaceEntry) GetFileCount() uint32 { + if m != nil { + return m.FileCount + } + return 0 +} + +func (m *SpaceEntry) GetCidCount() uint64 { + if m != nil { + return m.CidCount + } + return 0 +} + +type FileEntry struct { + Cids []string `protobuf:"bytes,1,rep,name=cids,proto3" json:"cids,omitempty"` + Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + CreateTime int64 `protobuf:"varint,3,opt,name=createTime,proto3" json:"createTime,omitempty"` + UpdateTime int64 `protobuf:"varint,4,opt,name=updateTime,proto3" json:"updateTime,omitempty"` +} + +func (m *FileEntry) Reset() { *m = FileEntry{} } +func (m *FileEntry) String() string { return proto.CompactTextString(m) } +func (*FileEntry) ProtoMessage() {} +func (*FileEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_f1f29953df8d243b, []int{4} +} +func (m *FileEntry) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FileEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FileEntry.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FileEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileEntry.Merge(m, src) +} +func (m *FileEntry) XXX_Size() int { + return m.Size() +} +func (m *FileEntry) XXX_DiscardUnknown() { + xxx_messageInfo_FileEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_FileEntry proto.InternalMessageInfo + +func (m *FileEntry) GetCids() []string { + if m != nil { + return m.Cids + } + return nil +} + +func (m *FileEntry) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func (m *FileEntry) GetCreateTime() int64 { + if m != nil { + return m.CreateTime + } + return 0 +} + +func (m *FileEntry) GetUpdateTime() int64 { + if m != nil { + return m.UpdateTime + } + return 0 +} + +func init() { + proto.RegisterType((*CidEntry)(nil), "fileIndexProto.CidEntry") + proto.RegisterType((*CidList)(nil), "fileIndexProto.CidList") + proto.RegisterType((*GroupEntry)(nil), "fileIndexProto.GroupEntry") + proto.RegisterType((*SpaceEntry)(nil), "fileIndexProto.SpaceEntry") + proto.RegisterType((*FileEntry)(nil), "fileIndexProto.FileEntry") +} + +func init() { + proto.RegisterFile("index/indexproto/protos/index.proto", fileDescriptor_f1f29953df8d243b) +} + +var fileDescriptor_f1f29953df8d243b = []byte{ + // 342 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x92, 0xcf, 0x4a, 0xc3, 0x40, + 0x10, 0xc6, 0x3b, 0x4d, 0xfa, 0x27, 0x83, 0x15, 0xc9, 0x69, 0x11, 0x5d, 0x42, 0xbc, 0x2c, 0x1e, + 0xec, 0xc1, 0x37, 0xb0, 0xa8, 0x14, 0x3c, 0xc8, 0xea, 0xc9, 0x5b, 0xcd, 0x6e, 0x65, 0xa1, 0x26, + 0x61, 0x37, 0x15, 0xf5, 0x19, 0x3c, 0xf8, 0x2c, 0xe2, 0x43, 0x78, 0xec, 0xd1, 0xa3, 0xb4, 0x2f, + 0x22, 0xbb, 0xa1, 0x49, 0x5b, 0x10, 0x2f, 0x82, 0x97, 0xcd, 0xfc, 0xbe, 0x21, 0xc9, 0xf7, 0xed, + 0x0c, 0x1e, 0xa8, 0x54, 0xc8, 0xc7, 0xbe, 0x3b, 0x73, 0x9d, 0x15, 0x59, 0xdf, 0x9d, 0xa6, 0x54, + 0x8e, 0x1c, 0x84, 0xdb, 0x63, 0x35, 0x91, 0x43, 0x2b, 0x5c, 0x5a, 0x8e, 0x5f, 0x00, 0xbb, 0x03, + 0x25, 0x4e, 0xd3, 0x42, 0x3f, 0x85, 0x21, 0xfa, 0x46, 0x3d, 0x4b, 0x02, 0x11, 0x30, 0x9f, 0xbb, + 0x3a, 0xa4, 0x88, 0x89, 0x96, 0xa3, 0x42, 0x5e, 0xab, 0x7b, 0x49, 0x9a, 0x11, 0x30, 0x8f, 0xaf, + 0x28, 0xb6, 0x3f, 0xcd, 0xc5, 0xb2, 0xef, 0x95, 0xfd, 0x5a, 0xb1, 0xdf, 0xd4, 0x72, 0x6c, 0x88, + 0x1f, 0x01, 0x6b, 0x71, 0x57, 0x87, 0x04, 0x3b, 0x0f, 0x52, 0x1b, 0x95, 0xa5, 0xa4, 0x15, 0x01, + 0xeb, 0xf1, 0x25, 0xc6, 0xfb, 0xd8, 0x19, 0x28, 0x71, 0xa1, 0x4c, 0x61, 0x5f, 0x4c, 0x94, 0x30, + 0x04, 0x22, 0x8f, 0x6d, 0x71, 0x57, 0xc7, 0x6f, 0x80, 0x78, 0xae, 0xb3, 0x69, 0x5e, 0xfa, 0x25, + 0xd8, 0xb9, 0xb3, 0x34, 0x14, 0xce, 0x72, 0xc0, 0x97, 0xf8, 0x17, 0xae, 0xdd, 0x4d, 0xf8, 0x2b, + 0x37, 0xb1, 0x8b, 0xdd, 0x44, 0x89, 0x41, 0x36, 0x4d, 0x0b, 0x67, 0xdb, 0xe7, 0x15, 0xdb, 0x9e, + 0xc9, 0x47, 0x89, 0x1c, 0x0a, 0x43, 0xda, 0x91, 0xc7, 0x02, 0x5e, 0x71, 0xfc, 0x0e, 0x88, 0x57, + 0x16, 0xfe, 0xc3, 0xf4, 0x1e, 0x06, 0x76, 0xe2, 0xb5, 0xeb, 0x1e, 0xaf, 0x85, 0xb5, 0x48, 0xed, + 0xf5, 0x48, 0xb1, 0xc1, 0xe0, 0x4c, 0x4d, 0x64, 0xb5, 0x19, 0xd5, 0x30, 0x82, 0x72, 0x18, 0xd5, + 0xef, 0x9a, 0x3f, 0x6e, 0x8b, 0xf7, 0x4b, 0x04, 0x7f, 0x33, 0xc2, 0xc9, 0xe1, 0xc7, 0x9c, 0xc2, + 0x6c, 0x4e, 0xe1, 0x6b, 0x4e, 0xe1, 0x75, 0x41, 0x1b, 0xb3, 0x05, 0x6d, 0x7c, 0x2e, 0x68, 0xe3, + 0x66, 0x67, 0x73, 0xbb, 0x6f, 0xdb, 0xee, 0x71, 0xfc, 0x1d, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xc4, + 0xb2, 0x7d, 0xf8, 0x02, 0x00, 0x00, +} + +func (m *CidEntry) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CidEntry) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CidEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Version != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Version)) + i-- + dAtA[i] = 0x28 + } + if m.Refs != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Refs)) + i-- + dAtA[i] = 0x20 + } + if m.UpdateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.UpdateTime)) + i-- + dAtA[i] = 0x18 + } + if m.CreateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CreateTime)) + i-- + dAtA[i] = 0x10 + } + if m.Size_ != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *CidList) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CidList) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CidList) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Cids) > 0 { + for iNdEx := len(m.Cids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Cids[iNdEx]) + copy(dAtA[i:], m.Cids[iNdEx]) + i = encodeVarintIndex(dAtA, i, uint64(len(m.Cids[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *GroupEntry) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GroupEntry) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GroupEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.SpaceIds) > 0 { + for iNdEx := len(m.SpaceIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceIds[iNdEx]) + copy(dAtA[i:], m.SpaceIds[iNdEx]) + i = encodeVarintIndex(dAtA, i, uint64(len(m.SpaceIds[iNdEx]))) + i-- + dAtA[i] = 0x32 + } + } + if m.CidCount != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CidCount)) + i-- + dAtA[i] = 0x28 + } + if m.Size_ != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x20 + } + if m.UpdateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.UpdateTime)) + i-- + dAtA[i] = 0x18 + } + if m.CreateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CreateTime)) + i-- + dAtA[i] = 0x10 + } + if len(m.GroupId) > 0 { + i -= len(m.GroupId) + copy(dAtA[i:], m.GroupId) + i = encodeVarintIndex(dAtA, i, uint64(len(m.GroupId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceEntry) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceEntry) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SpaceEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.CidCount != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CidCount)) + i-- + dAtA[i] = 0x30 + } + if m.FileCount != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.FileCount)) + i-- + dAtA[i] = 0x28 + } + if m.Size_ != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x20 + } + if m.UpdateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.UpdateTime)) + i-- + dAtA[i] = 0x18 + } + if m.CreateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CreateTime)) + i-- + dAtA[i] = 0x10 + } + if len(m.GroupId) > 0 { + i -= len(m.GroupId) + copy(dAtA[i:], m.GroupId) + i = encodeVarintIndex(dAtA, i, uint64(len(m.GroupId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileEntry) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileEntry) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FileEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.UpdateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.UpdateTime)) + i-- + dAtA[i] = 0x20 + } + if m.CreateTime != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.CreateTime)) + i-- + dAtA[i] = 0x18 + } + if m.Size_ != 0 { + i = encodeVarintIndex(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x10 + } + if len(m.Cids) > 0 { + for iNdEx := len(m.Cids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Cids[iNdEx]) + copy(dAtA[i:], m.Cids[iNdEx]) + i = encodeVarintIndex(dAtA, i, uint64(len(m.Cids[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func encodeVarintIndex(dAtA []byte, offset int, v uint64) int { + offset -= sovIndex(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CidEntry) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Size_ != 0 { + n += 1 + sovIndex(uint64(m.Size_)) + } + if m.CreateTime != 0 { + n += 1 + sovIndex(uint64(m.CreateTime)) + } + if m.UpdateTime != 0 { + n += 1 + sovIndex(uint64(m.UpdateTime)) + } + if m.Refs != 0 { + n += 1 + sovIndex(uint64(m.Refs)) + } + if m.Version != 0 { + n += 1 + sovIndex(uint64(m.Version)) + } + return n +} + +func (m *CidList) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Cids) > 0 { + for _, b := range m.Cids { + l = len(b) + n += 1 + l + sovIndex(uint64(l)) + } + } + return n +} + +func (m *GroupEntry) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.GroupId) + if l > 0 { + n += 1 + l + sovIndex(uint64(l)) + } + if m.CreateTime != 0 { + n += 1 + sovIndex(uint64(m.CreateTime)) + } + if m.UpdateTime != 0 { + n += 1 + sovIndex(uint64(m.UpdateTime)) + } + if m.Size_ != 0 { + n += 1 + sovIndex(uint64(m.Size_)) + } + if m.CidCount != 0 { + n += 1 + sovIndex(uint64(m.CidCount)) + } + if len(m.SpaceIds) > 0 { + for _, s := range m.SpaceIds { + l = len(s) + n += 1 + l + sovIndex(uint64(l)) + } + } + return n +} + +func (m *SpaceEntry) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.GroupId) + if l > 0 { + n += 1 + l + sovIndex(uint64(l)) + } + if m.CreateTime != 0 { + n += 1 + sovIndex(uint64(m.CreateTime)) + } + if m.UpdateTime != 0 { + n += 1 + sovIndex(uint64(m.UpdateTime)) + } + if m.Size_ != 0 { + n += 1 + sovIndex(uint64(m.Size_)) + } + if m.FileCount != 0 { + n += 1 + sovIndex(uint64(m.FileCount)) + } + if m.CidCount != 0 { + n += 1 + sovIndex(uint64(m.CidCount)) + } + return n +} + +func (m *FileEntry) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Cids) > 0 { + for _, s := range m.Cids { + l = len(s) + n += 1 + l + sovIndex(uint64(l)) + } + } + if m.Size_ != 0 { + n += 1 + sovIndex(uint64(m.Size_)) + } + if m.CreateTime != 0 { + n += 1 + sovIndex(uint64(m.CreateTime)) + } + if m.UpdateTime != 0 { + n += 1 + sovIndex(uint64(m.UpdateTime)) + } + return n +} + +func sovIndex(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozIndex(x uint64) (n int) { + return sovIndex(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *CidEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CidEntry: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CidEntry: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateTime", wireType) + } + m.CreateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateTime", wireType) + } + m.UpdateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Refs", wireType) + } + m.Refs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Refs |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + m.Version = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Version |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIndex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CidList) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CidList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CidList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthIndex + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthIndex + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cids = append(m.Cids, make([]byte, postIndex-iNdEx)) + copy(m.Cids[len(m.Cids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIndex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GroupEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GroupEntry: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GroupEntry: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthIndex + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthIndex + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GroupId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateTime", wireType) + } + m.CreateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateTime", wireType) + } + m.UpdateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CidCount", wireType) + } + m.CidCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CidCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthIndex + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthIndex + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceIds = append(m.SpaceIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIndex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceEntry: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceEntry: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthIndex + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthIndex + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GroupId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateTime", wireType) + } + m.CreateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateTime", wireType) + } + m.UpdateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FileCount", wireType) + } + m.FileCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FileCount |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CidCount", wireType) + } + m.CidCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CidCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIndex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileEntry: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileEntry: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cids", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthIndex + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthIndex + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cids = append(m.Cids, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreateTime", wireType) + } + m.CreateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateTime", wireType) + } + m.UpdateTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdateTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthIndex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipIndex(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIndex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIndex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowIndex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthIndex + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupIndex + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthIndex + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthIndex = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowIndex = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupIndex = fmt.Errorf("proto: unexpected end of group") +) diff --git a/index/indexproto/protos/index.proto b/index/indexproto/protos/index.proto new file mode 100644 index 00000000..e220e2a1 --- /dev/null +++ b/index/indexproto/protos/index.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; +package fileIndexProto; + +option go_package = "index/indexproto"; + +message CidEntry { + uint64 size = 1; + int64 createTime = 2; + int64 updateTime = 3; + int32 refs = 4; + uint32 version = 5; +} + +message CidList { + repeated bytes cids = 1; +} + +message GroupEntry { + string groupId = 1; + int64 createTime = 2; + int64 updateTime = 3; + uint64 size = 4; + uint64 cidCount = 5; + repeated string spaceIds = 6; +} + +message SpaceEntry { + string groupId = 1; + int64 createTime = 2; + int64 updateTime = 3; + uint64 size = 4; + uint32 fileCount = 5; + uint64 cidCount = 6; +} + +message FileEntry { + repeated string cids = 1; + uint64 size = 2; + int64 createTime = 3; + int64 updateTime = 4; +} \ No newline at end of file diff --git a/index/loader.go b/index/loader.go new file mode 100644 index 00000000..0888b436 --- /dev/null +++ b/index/loader.go @@ -0,0 +1,238 @@ +package index + +import ( + "context" + "errors" + "math/rand" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/cespare/xxhash/v2" + "github.com/go-redsync/redsync/v4" + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +const ( + partitionCount = 256 + + persistThreads = 10 +) + +var partitions = make([]int, partitionCount) + +func init() { + for i := range partitions { + partitions[i] = i + } +} + +type persistentStore interface { + IndexGet(ctx context.Context, key string) (value []byte, err error) + IndexPut(ctx context.Context, key string, value []byte) (err error) +} + +func bloomFilterKey(key string) string { + sum := xxhash.Sum64String(key) % partitionCount + return "bf:{" + strconv.FormatUint(sum, 10) + "}" +} + +func storeKey(key string) string { + sum := xxhash.Sum64String(key) % partitionCount + return "store:{" + strconv.FormatUint(sum, 10) + "}" +} + +func (ri *redisIndex) CheckKey(ctx context.Context, key string) (exists bool, err error) { + var release func() + if exists, release, err = ri.acquireKey(ctx, key); err != nil { + return + } + release() + return +} + +func (ri *redisIndex) AcquireKey(ctx context.Context, key string) (exists bool, release func(), err error) { + if exists, release, err = ri.acquireKey(ctx, key); err != nil { + return + } + if err = ri.updateKeyUsage(ctx, key); err != nil { + release() + return false, nil, err + } + return +} + +func (ri *redisIndex) acquireKey(ctx context.Context, key string) (exists bool, release func(), err error) { + mu := ri.redsync.NewMutex("_lock:"+key, redsync.WithExpiry(time.Minute)) + if err = mu.LockContext(ctx); err != nil { + return + } + release = func() { + _, _ = mu.Unlock() + } + + // update activity by key + if err = ri.updateKeyUsage(ctx, key); err != nil { + release() + return false, nil, err + } + + // check in redis + ex, err := ri.cl.Exists(ctx, key).Result() + if err != nil { + release() + return + } + // already in redis + if ex > 0 { + return true, release, nil + } + + // check bloom filter + bfKey := bloomFilterKey(key) + bloomEx, err := ri.cl.BFExists(ctx, bfKey, key).Result() + if err != nil { + release() + return false, nil, err + } + // not in bloom filter, item not exists + if !bloomEx { + return false, release, nil + } + + // try to load from persistent store + val, err := ri.persistStore.IndexGet(ctx, key) + if err != nil { + release() + return false, nil, err + } + // nil means not found + if val == nil { + return false, release, nil + } + if err = ri.cl.Restore(ctx, key, 0, string(val)).Err(); err != nil { + release() + return false, nil, err + } + return true, release, nil +} +func (ri *redisIndex) updateKeyUsage(ctx context.Context, key string) (err error) { + sKey := storeKey(key) + return ri.cl.ZAdd(ctx, sKey, redis.Z{ + Score: float64(time.Now().Unix()), + Member: key, + }).Err() +} + +func (ri *redisIndex) PersistKeys(ctx context.Context) { + st := time.Now() + rand.Shuffle(len(partitions), func(i, j int) { + partitions[i], partitions[j] = partitions[j], partitions[i] + }) + stat := &persistStat{} + var wg sync.WaitGroup + wg.Add(len(partitions)) + var limiter = make(chan struct{}, persistThreads) + for _, part := range partitions { + limiter <- struct{}{} + go func(p int) { + defer func() { + <-limiter + wg.Done() + }() + if e := ri.persistKeys(ctx, p, stat); e != nil { + log.Warn("persist part error", zap.Error(e), zap.Int("part", p)) + } + }(part) + } + wg.Wait() + log.Info("persist", + zap.Duration("dur", time.Since(st)), + zap.Int32("handled", stat.handled.Load()), + zap.Int32("deleted", stat.deleted.Load()), + zap.Int32("missed", stat.missed.Load()), + zap.Int32("errors", stat.errors.Load()), + zap.Int32("moved", stat.moved.Load()), + zap.Int32("moved kbs", stat.movedBytes.Load()/1024), + ) +} + +func (ri *redisIndex) persistKeys(ctx context.Context, part int, stat *persistStat) (err error) { + deadline := time.Now().Add(-ri.persistTtl).Unix() + sk := "store:{" + strconv.FormatInt(int64(part), 10) + "}" + keys, err := ri.cl.ZRangeByScore(ctx, sk, &redis.ZRangeBy{ + Min: "0", + Max: strconv.FormatInt(deadline, 10), + }).Result() + if err != nil { + return + } + for _, k := range keys { + if err = ri.persistKey(ctx, sk, k, deadline, stat); err != nil { + return + } + } + return +} + +func (ri *redisIndex) persistKey(ctx context.Context, storeKey, key string, deadline int64, stat *persistStat) (err error) { + mu := ri.redsync.NewMutex("_lock:" + key) + if err = mu.LockContext(ctx); err != nil { + return + } + defer func() { + _, _ = mu.Unlock() + }() + + stat.handled.Add(1) + + // make sure lastActivity not changed + res, err := ri.cl.ZMScore(ctx, storeKey, key).Result() + if err != nil { + stat.errors.Add(1) + return + } + if int64(res[0]) > deadline { + stat.missed.Add(1) + return + } + + dump, err := ri.cl.Dump(ctx, key).Result() + if err != nil && !errors.Is(err, redis.Nil) { + stat.errors.Add(1) + return + } + // key was removed - just remove it from store queue + if errors.Is(err, redis.Nil) { + stat.deleted.Add(1) + return ri.cl.ZRem(ctx, storeKey, key).Err() + } + + // persist the dump + if err = ri.persistStore.IndexPut(ctx, key, []byte(dump)); err != nil { + return + } + // remove from queue and add to bloom filter + _, err = ri.cl.TxPipelined(ctx, func(tx redis.Pipeliner) error { + tx.ZRem(ctx, storeKey, key) + tx.BFAdd(ctx, bloomFilterKey(key), key) + return nil + }) + + stat.moved.Add(1) + stat.movedBytes.Add(int32(len(dump))) + + // remove key + return ri.cl.Del(ctx, key).Err() +} + +type persistStat struct { + handled atomic.Int32 + moved atomic.Int32 + movedBytes atomic.Int32 + missed atomic.Int32 + deleted atomic.Int32 + errors atomic.Int32 +} diff --git a/index/loader_test.go b/index/loader_test.go new file mode 100644 index 00000000..1d1c5126 --- /dev/null +++ b/index/loader_test.go @@ -0,0 +1,77 @@ +package index + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_PersistKeys(t *testing.T) { + t.Run("no keys", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + fx.persistTtl = time.Second * 2 + bs := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + fx.PersistKeys(ctx) + }) + t.Run("persist", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + fx.persistTtl = time.Second + bs := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + for _, b := range bs { + fx.persistStore.EXPECT().IndexPut(ctx, cidKey(b.Cid()), gomock.Any()) + } + + time.Sleep(time.Second * 3) + bs2 := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs2)) + + fx.PersistKeys(ctx) + + for _, b := range bs { + res, err := fx.cl.BFExists(ctx, bloomFilterKey(cidKey(b.Cid())), cidKey(b.Cid())).Result() + require.NoError(t, err) + assert.True(t, res) + } + }) +} + +func TestRedisIndex_AcquireKey(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + fx.persistTtl = time.Second + bs := testutil.NewRandBlocks(5) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + for _, b := range bs { + fx.persistStore.EXPECT().IndexPut(ctx, cidKey(b.Cid()), gomock.Any()).Do(func(_ context.Context, key string, value []byte) { + if key == cidKey(bs[0].Cid()) { + fx.persistStore.EXPECT().IndexGet(ctx, key).Return(nil, nil) + } else { + fx.persistStore.EXPECT().IndexGet(ctx, key).Return(value, nil) + } + }) + } + time.Sleep(time.Second * 3) + fx.PersistKeys(ctx) + + for i, b := range bs { + ex, release, err := fx.AcquireKey(ctx, cidKey(b.Cid())) + require.NoError(t, err) + if i == 0 { + require.False(t, ex) + } else { + require.True(t, ex) + } + release() + } + +} diff --git a/index/migrate.go b/index/migrate.go new file mode 100644 index 00000000..4885a80b --- /dev/null +++ b/index/migrate.go @@ -0,0 +1,106 @@ +package index + +import ( + "context" + "strings" + "time" + + "github.com/go-redsync/redsync/v4" + "github.com/golang/snappy" + "github.com/ipfs/go-cid" + "go.uber.org/zap" + + "github.com/anyproto/any-sync-filenode/index/indexproto" +) + +func (ri *redisIndex) Migrate(ctx context.Context, key Key) (err error) { + st := time.Now() + // fast check before lock the key + var checkKeys = []string{ + "s:" + key.SpaceId, + "s:" + key.GroupId, + } + var migrateKey string + for _, checkKey := range checkKeys { + ex, err := ri.cl.Exists(ctx, checkKey).Result() + if err != nil { + return err + } + if ex != 0 { + migrateKey = checkKey + break + } + } + // old keys doesn't exist + if migrateKey == "" { + return + } + + // lock the key + mu := ri.redsync.NewMutex("_lock:"+migrateKey, redsync.WithExpiry(time.Minute)) + if err = mu.LockContext(ctx); err != nil { + return + } + defer func() { + _, _ = mu.Unlock() + }() + + // another check under lock + ex, err := ri.cl.Exists(ctx, migrateKey).Result() + if err != nil { + return + } + if ex == 0 { + // key doesn't exist - another node did the migration + return + } + + keys, err := ri.cl.HKeys(ctx, migrateKey).Result() + if err != nil { + return + } + + for _, k := range keys { + if strings.HasPrefix(k, "f:") { + if err = ri.migrateFile(ctx, key, migrateKey, k); err != nil { + return + } + } + } + if err = ri.cl.Del(ctx, migrateKey).Err(); err != nil { + return + } + log.Info("space migrated", zap.String("spaceId", key.SpaceId), zap.Duration("dur", time.Since(st))) + return +} + +func (ri *redisIndex) migrateFile(ctx context.Context, key Key, migrateKey, fileKey string) (err error) { + data, err := ri.cl.HGet(ctx, migrateKey, fileKey).Result() + if err != nil { + return + } + + encodedData, err := snappy.Decode(nil, []byte(data)) + if err != nil { + return + } + oldList := &indexproto.CidList{} + if err = oldList.Unmarshal(encodedData); err != nil { + return + } + + var cidList = make([]cid.Cid, len(oldList.Cids)) + for i, cb := range oldList.Cids { + if cidList[i], err = cid.Cast(cb); err != nil { + return + } + } + + cidEntries, err := ri.CidEntries(ctx, cidList) + if err != nil { + return + } + defer cidEntries.Release() + + return ri.FileBind(ctx, key, fileKey[2:], cidEntries) +} diff --git a/index/migrate_test.go b/index/migrate_test.go new file mode 100644 index 00000000..d76399c0 --- /dev/null +++ b/index/migrate_test.go @@ -0,0 +1,38 @@ +package index + +import ( + "archive/zip" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRedisIndex_Migrate(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + + // load old space keys + zrd, err := zip.OpenReader("testdata/oldspace.zip") + require.NoError(t, err) + defer zrd.Close() + for _, zf := range zrd.File { + zfr, err := zf.Open() + require.NoError(t, err) + data, err := io.ReadAll(zfr) + require.NoError(t, err) + require.NoError(t, fx.cl.Restore(ctx, zf.Name, 0, string(data)).Err()) + _ = zfr.Close() + } + + // migrate + expectedSize := uint64(18248267) + key := newRandKey() + key.SpaceId = "bafyreic65hvluhooz7u43hniptb4uokmpaqm6b2aneym77pivurjt4csze.2e2j1mpearah" + require.NoError(t, fx.Migrate(ctx, key)) + + spaceInfo, err := fx.SpaceInfo(ctx, key) + require.NoError(t, err) + assert.Equal(t, expectedSize, spaceInfo.BytesUsage) +} diff --git a/index/mock_index/mock_index.go b/index/mock_index/mock_index.go index bd82cbed..66653489 100644 --- a/index/mock_index/mock_index.go +++ b/index/mock_index/mock_index.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/any-sync-filenode/index (interfaces: Index) - +// +// Generated by this command: +// +// mockgen -destination mock_index/mock_index.go github.com/anyproto/any-sync-filenode/index Index +// // Package mock_index is a generated GoMock package. package mock_index @@ -38,164 +42,218 @@ func (m *MockIndex) EXPECT() *MockIndexMockRecorder { return m.recorder } -// AddBlocks mocks base method. -func (m *MockIndex) AddBlocks(arg0 context.Context, arg1 []blocks.Block) error { +// BlocksAdd mocks base method. +func (m *MockIndex) BlocksAdd(arg0 context.Context, arg1 []blocks.Block) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddBlocks", arg0, arg1) + ret := m.ctrl.Call(m, "BlocksAdd", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } -// AddBlocks indicates an expected call of AddBlocks. -func (mr *MockIndexMockRecorder) AddBlocks(arg0, arg1 interface{}) *gomock.Call { +// BlocksAdd indicates an expected call of BlocksAdd. +func (mr *MockIndexMockRecorder) BlocksAdd(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddBlocks", reflect.TypeOf((*MockIndex)(nil).AddBlocks), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlocksAdd", reflect.TypeOf((*MockIndex)(nil).BlocksAdd), arg0, arg1) } -// Bind mocks base method. -func (m *MockIndex) Bind(arg0 context.Context, arg1, arg2 string, arg3 []blocks.Block) error { +// BlocksGetNonExistent mocks base method. +func (m *MockIndex) BlocksGetNonExistent(arg0 context.Context, arg1 []blocks.Block) ([]blocks.Block, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Bind", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "BlocksGetNonExistent", arg0, arg1) + ret0, _ := ret[0].([]blocks.Block) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// Bind indicates an expected call of Bind. -func (mr *MockIndexMockRecorder) Bind(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +// BlocksGetNonExistent indicates an expected call of BlocksGetNonExistent. +func (mr *MockIndexMockRecorder) BlocksGetNonExistent(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bind", reflect.TypeOf((*MockIndex)(nil).Bind), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlocksGetNonExistent", reflect.TypeOf((*MockIndex)(nil).BlocksGetNonExistent), arg0, arg1) } -// BindCids mocks base method. -func (m *MockIndex) BindCids(arg0 context.Context, arg1, arg2 string, arg3 []cid.Cid) error { +// BlocksLock mocks base method. +func (m *MockIndex) BlocksLock(arg0 context.Context, arg1 []blocks.Block) (func(), error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BindCids", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "BlocksLock", arg0, arg1) + ret0, _ := ret[0].(func()) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// BindCids indicates an expected call of BindCids. -func (mr *MockIndexMockRecorder) BindCids(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +// BlocksLock indicates an expected call of BlocksLock. +func (mr *MockIndexMockRecorder) BlocksLock(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindCids", reflect.TypeOf((*MockIndex)(nil).BindCids), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlocksLock", reflect.TypeOf((*MockIndex)(nil).BlocksLock), arg0, arg1) } -// Exists mocks base method. -func (m *MockIndex) Exists(arg0 context.Context, arg1 cid.Cid) (bool, error) { +// CidEntries mocks base method. +func (m *MockIndex) CidEntries(arg0 context.Context, arg1 []cid.Cid) (*index.CidEntries, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Exists", arg0, arg1) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "CidEntries", arg0, arg1) + ret0, _ := ret[0].(*index.CidEntries) ret1, _ := ret[1].(error) return ret0, ret1 } -// Exists indicates an expected call of Exists. -func (mr *MockIndexMockRecorder) Exists(arg0, arg1 interface{}) *gomock.Call { +// CidEntries indicates an expected call of CidEntries. +func (mr *MockIndexMockRecorder) CidEntries(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockIndex)(nil).Exists), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CidEntries", reflect.TypeOf((*MockIndex)(nil).CidEntries), arg0, arg1) } -// ExistsInStorage mocks base method. -func (m *MockIndex) ExistsInStorage(arg0 context.Context, arg1 string, arg2 []cid.Cid) ([]cid.Cid, error) { +// CidEntriesByBlocks mocks base method. +func (m *MockIndex) CidEntriesByBlocks(arg0 context.Context, arg1 []blocks.Block) (*index.CidEntries, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExistsInStorage", arg0, arg1, arg2) - ret0, _ := ret[0].([]cid.Cid) + ret := m.ctrl.Call(m, "CidEntriesByBlocks", arg0, arg1) + ret0, _ := ret[0].(*index.CidEntries) ret1, _ := ret[1].(error) return ret0, ret1 } -// ExistsInStorage indicates an expected call of ExistsInStorage. -func (mr *MockIndexMockRecorder) ExistsInStorage(arg0, arg1, arg2 interface{}) *gomock.Call { +// CidEntriesByBlocks indicates an expected call of CidEntriesByBlocks. +func (mr *MockIndexMockRecorder) CidEntriesByBlocks(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistsInStorage", reflect.TypeOf((*MockIndex)(nil).ExistsInStorage), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CidEntriesByBlocks", reflect.TypeOf((*MockIndex)(nil).CidEntriesByBlocks), arg0, arg1) } -// FileInfo mocks base method. -func (m *MockIndex) FileInfo(arg0 context.Context, arg1, arg2 string) (index.FileInfo, error) { +// CidExists mocks base method. +func (m *MockIndex) CidExists(arg0 context.Context, arg1 cid.Cid) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FileInfo", arg0, arg1, arg2) - ret0, _ := ret[0].(index.FileInfo) + ret := m.ctrl.Call(m, "CidExists", arg0, arg1) + ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// FileInfo indicates an expected call of FileInfo. -func (mr *MockIndexMockRecorder) FileInfo(arg0, arg1, arg2 interface{}) *gomock.Call { +// CidExists indicates an expected call of CidExists. +func (mr *MockIndexMockRecorder) CidExists(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileInfo", reflect.TypeOf((*MockIndex)(nil).FileInfo), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CidExists", reflect.TypeOf((*MockIndex)(nil).CidExists), arg0, arg1) } -// GetNonExistentBlocks mocks base method. -func (m *MockIndex) GetNonExistentBlocks(arg0 context.Context, arg1 []blocks.Block) ([]blocks.Block, error) { +// CidExistsInSpace mocks base method. +func (m *MockIndex) CidExistsInSpace(arg0 context.Context, arg1 index.Key, arg2 []cid.Cid) ([]cid.Cid, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetNonExistentBlocks", arg0, arg1) - ret0, _ := ret[0].([]blocks.Block) + ret := m.ctrl.Call(m, "CidExistsInSpace", arg0, arg1, arg2) + ret0, _ := ret[0].([]cid.Cid) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetNonExistentBlocks indicates an expected call of GetNonExistentBlocks. -func (mr *MockIndexMockRecorder) GetNonExistentBlocks(arg0, arg1 interface{}) *gomock.Call { +// CidExistsInSpace indicates an expected call of CidExistsInSpace. +func (mr *MockIndexMockRecorder) CidExistsInSpace(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNonExistentBlocks", reflect.TypeOf((*MockIndex)(nil).GetNonExistentBlocks), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CidExistsInSpace", reflect.TypeOf((*MockIndex)(nil).CidExistsInSpace), arg0, arg1, arg2) } -// Init mocks base method. -func (m *MockIndex) Init(arg0 *app.App) error { +// Close mocks base method. +func (m *MockIndex) Close(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0) + ret := m.ctrl.Call(m, "Close", arg0) ret0, _ := ret[0].(error) return ret0 } -// Init indicates an expected call of Init. -func (mr *MockIndexMockRecorder) Init(arg0 interface{}) *gomock.Call { +// Close indicates an expected call of Close. +func (mr *MockIndexMockRecorder) Close(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockIndex)(nil).Init), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockIndex)(nil).Close), arg0) } -// IsAllExists mocks base method. -func (m *MockIndex) IsAllExists(arg0 context.Context, arg1 []cid.Cid) (bool, error) { +// FileBind mocks base method. +func (m *MockIndex) FileBind(arg0 context.Context, arg1 index.Key, arg2 string, arg3 *index.CidEntries) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsAllExists", arg0, arg1) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "FileBind", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// FileBind indicates an expected call of FileBind. +func (mr *MockIndexMockRecorder) FileBind(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileBind", reflect.TypeOf((*MockIndex)(nil).FileBind), arg0, arg1, arg2, arg3) +} + +// FileInfo mocks base method. +func (m *MockIndex) FileInfo(arg0 context.Context, arg1 index.Key, arg2 ...string) ([]index.FileInfo, error) { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FileInfo", varargs...) + ret0, _ := ret[0].([]index.FileInfo) ret1, _ := ret[1].(error) return ret0, ret1 } -// IsAllExists indicates an expected call of IsAllExists. -func (mr *MockIndexMockRecorder) IsAllExists(arg0, arg1 interface{}) *gomock.Call { +// FileInfo indicates an expected call of FileInfo. +func (mr *MockIndexMockRecorder) FileInfo(arg0, arg1 any, arg2 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileInfo", reflect.TypeOf((*MockIndex)(nil).FileInfo), varargs...) +} + +// FileUnbind mocks base method. +func (m *MockIndex) FileUnbind(arg0 context.Context, arg1 index.Key, arg2 ...string) error { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FileUnbind", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// FileUnbind indicates an expected call of FileUnbind. +func (mr *MockIndexMockRecorder) FileUnbind(arg0, arg1 any, arg2 ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAllExists", reflect.TypeOf((*MockIndex)(nil).IsAllExists), arg0, arg1) + varargs := append([]any{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileUnbind", reflect.TypeOf((*MockIndex)(nil).FileUnbind), varargs...) } -// Lock mocks base method. -func (m *MockIndex) Lock(arg0 context.Context, arg1 []cid.Cid) (func(), error) { +// GroupInfo mocks base method. +func (m *MockIndex) GroupInfo(arg0 context.Context, arg1 string) (index.GroupInfo, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Lock", arg0, arg1) - ret0, _ := ret[0].(func()) + ret := m.ctrl.Call(m, "GroupInfo", arg0, arg1) + ret0, _ := ret[0].(index.GroupInfo) ret1, _ := ret[1].(error) return ret0, ret1 } -// Lock indicates an expected call of Lock. -func (mr *MockIndexMockRecorder) Lock(arg0, arg1 interface{}) *gomock.Call { +// GroupInfo indicates an expected call of GroupInfo. +func (mr *MockIndexMockRecorder) GroupInfo(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GroupInfo", reflect.TypeOf((*MockIndex)(nil).GroupInfo), arg0, arg1) +} + +// Init mocks base method. +func (m *MockIndex) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockIndexMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockIndex)(nil).Lock), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockIndex)(nil).Init), arg0) } -// MoveStorage mocks base method. -func (m *MockIndex) MoveStorage(arg0 context.Context, arg1, arg2 string) error { +// Migrate mocks base method. +func (m *MockIndex) Migrate(arg0 context.Context, arg1 index.Key) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MoveStorage", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Migrate", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } -// MoveStorage indicates an expected call of MoveStorage. -func (mr *MockIndexMockRecorder) MoveStorage(arg0, arg1, arg2 interface{}) *gomock.Call { +// Migrate indicates an expected call of Migrate. +func (mr *MockIndexMockRecorder) Migrate(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockIndex)(nil).MoveStorage), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Migrate", reflect.TypeOf((*MockIndex)(nil).Migrate), arg0, arg1) } // Name mocks base method. @@ -212,46 +270,31 @@ func (mr *MockIndexMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockIndex)(nil).Name)) } -// StorageInfo mocks base method. -func (m *MockIndex) StorageInfo(arg0 context.Context, arg1 string) (index.StorageInfo, error) { +// Run mocks base method. +func (m *MockIndex) Run(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StorageInfo", arg0, arg1) - ret0, _ := ret[0].(index.StorageInfo) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "Run", arg0) + ret0, _ := ret[0].(error) + return ret0 } -// StorageInfo indicates an expected call of StorageInfo. -func (mr *MockIndexMockRecorder) StorageInfo(arg0, arg1 interface{}) *gomock.Call { +// Run indicates an expected call of Run. +func (mr *MockIndexMockRecorder) Run(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageInfo", reflect.TypeOf((*MockIndex)(nil).StorageInfo), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockIndex)(nil).Run), arg0) } -// StorageSize mocks base method. -func (m *MockIndex) StorageSize(arg0 context.Context, arg1 string) (uint64, error) { +// SpaceInfo mocks base method. +func (m *MockIndex) SpaceInfo(arg0 context.Context, arg1 index.Key) (index.SpaceInfo, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StorageSize", arg0, arg1) - ret0, _ := ret[0].(uint64) + ret := m.ctrl.Call(m, "SpaceInfo", arg0, arg1) + ret0, _ := ret[0].(index.SpaceInfo) ret1, _ := ret[1].(error) return ret0, ret1 } -// StorageSize indicates an expected call of StorageSize. -func (mr *MockIndexMockRecorder) StorageSize(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageSize", reflect.TypeOf((*MockIndex)(nil).StorageSize), arg0, arg1) -} - -// UnBind mocks base method. -func (m *MockIndex) UnBind(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnBind", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// UnBind indicates an expected call of UnBind. -func (mr *MockIndexMockRecorder) UnBind(arg0, arg1, arg2 interface{}) *gomock.Call { +// SpaceInfo indicates an expected call of SpaceInfo. +func (mr *MockIndexMockRecorder) SpaceInfo(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnBind", reflect.TypeOf((*MockIndex)(nil).UnBind), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceInfo", reflect.TypeOf((*MockIndex)(nil).SpaceInfo), arg0, arg1) } diff --git a/index/redisindex/bind.go b/index/redisindex/bind.go deleted file mode 100644 index 6bd9b4fe..00000000 --- a/index/redisindex/bind.go +++ /dev/null @@ -1,72 +0,0 @@ -package redisindex - -import ( - "context" - "github.com/redis/go-redis/v9" -) - -type bindOp struct { - sk string - fk string - ri *redisIndex -} - -func (op *bindOp) Bind(ctx context.Context, cids []CidInfo) (newCids []CidInfo, err error) { - // fetch existing or create new file cids list - fCids, err := op.ri.newFileCidList(ctx, op.sk, op.fk) - if err != nil { - return nil, err - } - defer fCids.Unlock() - - // make a list of indexes non-exists cids - var newInFileIdx = make([]int, 0, len(cids)) - for i, c := range cids { - if !fCids.Exists(c.Cid) { - newInFileIdx = append(newInFileIdx, i) - fCids.Add(c.Cid) - } - } - - // all cids exists, nothing to do - if len(newInFileIdx) == 0 { - return nil, nil - } - - // make a list of results, data will be available after executing of pipeline - var execResults = make([]*redis.IntCmd, len(newInFileIdx)) - // do updates in one pipeline - _, err = op.ri.cl.TxPipelined(ctx, func(pipe redis.Pipeliner) error { - for i, idx := range newInFileIdx { - ck := cids[idx].Cid.String() - execResults[i] = pipe.HIncrBy(ctx, op.sk, ck, 1) - if err = execResults[i].Err(); err != nil { - return err - } - } - return fCids.Save(ctx, pipe) - }) - if err != nil { - return - } - - // check for newly added cids - // calculate spaceSize increase - // make a list of finally added cids - var spaceIncreaseSize uint64 - for i, res := range execResults { - if newCounter, err := res.Result(); err == nil && newCounter == 1 { - c := cids[newInFileIdx[i]] - spaceIncreaseSize += c.Size_ - newCids = append(newCids, c) - } - } - - // increment space size - if spaceIncreaseSize > 0 { - if err = op.ri.cl.HIncrBy(ctx, op.sk, storeSizeKey, int64(spaceIncreaseSize)).Err(); err != nil { - return nil, err - } - } - return -} diff --git a/index/redisindex/bind_test.go b/index/redisindex/bind_test.go deleted file mode 100644 index cee8048d..00000000 --- a/index/redisindex/bind_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package redisindex - -import ( - "github.com/anyproto/any-sync-filenode/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" -) - -func TestRedisIndex_Bind(t *testing.T) { - t.Run("first add", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - bs := testutil.NewRandBlocks(3) - var sumSize uint64 - for _, b := range bs { - sumSize += uint64(len(b.RawData())) - } - spaceId := testutil.NewRandSpaceId() - fileId := testutil.NewRandCid().String() - - require.NoError(t, fx.Bind(ctx, spaceId, fileId, bs)) - size, err := fx.StorageSize(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, sumSize, size) - }) - t.Run("two files with same cids", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - bs := testutil.NewRandBlocks(3) - var sumSize uint64 - for _, b := range bs { - sumSize += uint64(len(b.RawData())) - } - spaceId := testutil.NewRandSpaceId() - fileId1 := testutil.NewRandCid().String() - fileId2 := testutil.NewRandCid().String() - - require.NoError(t, fx.Bind(ctx, spaceId, fileId1, bs[:2])) - require.NoError(t, fx.Bind(ctx, spaceId, fileId2, bs)) - size, err := fx.StorageSize(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, sumSize, size) - }) - t.Run("bind twice", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - bs := testutil.NewRandBlocks(3) - var sumSize uint64 - for _, b := range bs { - sumSize += uint64(len(b.RawData())) - } - spaceId := testutil.NewRandSpaceId() - fileId := testutil.NewRandCid().String() - - require.NoError(t, fx.Bind(ctx, spaceId, fileId, bs)) - require.NoError(t, fx.Bind(ctx, spaceId, fileId, bs)) - size, err := fx.StorageSize(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, sumSize, size) - }) -} - -func TestRedisIndex_BindCids(t *testing.T) { - t.Run("success", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - bs := testutil.NewRandBlocks(3) - var sumSize uint64 - for _, b := range bs { - sumSize += uint64(len(b.RawData())) - } - spaceId := testutil.NewRandSpaceId() - fileId1 := testutil.NewRandCid().String() - fileId2 := testutil.NewRandCid().String() - - require.NoError(t, fx.Bind(ctx, spaceId, fileId1, bs)) - require.NoError(t, fx.BindCids(ctx, spaceId, fileId2, testutil.BlocksToKeys(bs))) - size, err := fx.StorageSize(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, sumSize, size) - - fi1, err := fx.FileInfo(ctx, spaceId, fileId1) - require.NoError(t, err) - fi2, err := fx.FileInfo(ctx, spaceId, fileId2) - require.NoError(t, err) - - assert.Equal(t, uint32(len(bs)), fi1.CidCount) - assert.Equal(t, size, fi1.BytesUsage) - assert.Equal(t, fi1, fi2) - }) - t.Run("bind not existing cids", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - bs := testutil.NewRandBlocks(3) - var sumSize uint64 - for _, b := range bs { - sumSize += uint64(len(b.RawData())) - } - spaceId := testutil.NewRandSpaceId() - fileId1 := testutil.NewRandCid().String() - require.Error(t, fx.BindCids(ctx, spaceId, fileId1, testutil.BlocksToKeys(bs))) - }) -} diff --git a/index/redisindex/fcidlist.go b/index/redisindex/fcidlist.go deleted file mode 100644 index d5b34b8e..00000000 --- a/index/redisindex/fcidlist.go +++ /dev/null @@ -1,76 +0,0 @@ -package redisindex - -import ( - "bytes" - "context" - "github.com/anyproto/any-sync-filenode/index/redisindex/indexproto" - "github.com/go-redsync/redsync/v4" - "github.com/golang/snappy" - "github.com/ipfs/go-cid" - "github.com/redis/go-redis/v9" -) - -func (r *redisIndex) newFileCidList(ctx context.Context, sk, fk string) (l *fileCidList, err error) { - lock := r.redsync.NewMutex("_lock:fk:" + sk + ":" + fk) - if err = lock.LockContext(ctx); err != nil { - return nil, err - } - res, err := r.cl.HGet(ctx, sk, fk).Result() - if err == redis.Nil { - err = nil - } - if err != nil { - return - } - l = &fileCidList{ - CidList: &indexproto.CidList{}, - cl: r.cl, - sk: sk, - fk: fk, - lock: lock, - } - if len(res) == 0 { - return - } - data, err := snappy.Decode(nil, []byte(res)) - if err != nil { - return - } - if err = l.CidList.Unmarshal(data); err != nil { - return - } - return -} - -type fileCidList struct { - *indexproto.CidList - cl redis.UniversalClient - sk, fk string - lock *redsync.Mutex -} - -func (cl *fileCidList) Exists(k cid.Cid) bool { - for _, c := range cl.Cids { - if bytes.Equal(k.Bytes(), c) { - return true - } - } - return false -} - -func (cl *fileCidList) Add(k cid.Cid) { - cl.Cids = append(cl.Cids, k.Bytes()) -} - -func (cl *fileCidList) Unlock() { - _, _ = cl.lock.Unlock() -} - -func (cl *fileCidList) Save(ctx context.Context, tx redis.Pipeliner) (err error) { - if len(cl.Cids) == 0 { - return tx.HDel(ctx, cl.sk, cl.fk).Err() - } - data, _ := cl.Marshal() - data = snappy.Encode(nil, data) - return tx.HSet(ctx, cl.sk, cl.fk, data).Err() -} diff --git a/index/redisindex/indexproto/index.pb.go b/index/redisindex/indexproto/index.pb.go deleted file mode 100644 index 7655481c..00000000 --- a/index/redisindex/indexproto/index.pb.go +++ /dev/null @@ -1,582 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: index/redisindex/indexproto/protos/index.proto - -package indexproto - -import ( - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type CidEntry struct { - Size_ uint64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - CreateTime int64 `protobuf:"varint,2,opt,name=createTime,proto3" json:"createTime,omitempty"` - UpdateTime int64 `protobuf:"varint,3,opt,name=updateTime,proto3" json:"updateTime,omitempty"` - Refs int32 `protobuf:"varint,4,opt,name=refs,proto3" json:"refs,omitempty"` -} - -func (m *CidEntry) Reset() { *m = CidEntry{} } -func (m *CidEntry) String() string { return proto.CompactTextString(m) } -func (*CidEntry) ProtoMessage() {} -func (*CidEntry) Descriptor() ([]byte, []int) { - return fileDescriptor_01af1a9166444478, []int{0} -} -func (m *CidEntry) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CidEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CidEntry.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *CidEntry) XXX_Merge(src proto.Message) { - xxx_messageInfo_CidEntry.Merge(m, src) -} -func (m *CidEntry) XXX_Size() int { - return m.Size() -} -func (m *CidEntry) XXX_DiscardUnknown() { - xxx_messageInfo_CidEntry.DiscardUnknown(m) -} - -var xxx_messageInfo_CidEntry proto.InternalMessageInfo - -func (m *CidEntry) GetSize_() uint64 { - if m != nil { - return m.Size_ - } - return 0 -} - -func (m *CidEntry) GetCreateTime() int64 { - if m != nil { - return m.CreateTime - } - return 0 -} - -func (m *CidEntry) GetUpdateTime() int64 { - if m != nil { - return m.UpdateTime - } - return 0 -} - -func (m *CidEntry) GetRefs() int32 { - if m != nil { - return m.Refs - } - return 0 -} - -type CidList struct { - Cids [][]byte `protobuf:"bytes,1,rep,name=cids,proto3" json:"cids,omitempty"` -} - -func (m *CidList) Reset() { *m = CidList{} } -func (m *CidList) String() string { return proto.CompactTextString(m) } -func (*CidList) ProtoMessage() {} -func (*CidList) Descriptor() ([]byte, []int) { - return fileDescriptor_01af1a9166444478, []int{1} -} -func (m *CidList) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CidList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CidList.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *CidList) XXX_Merge(src proto.Message) { - xxx_messageInfo_CidList.Merge(m, src) -} -func (m *CidList) XXX_Size() int { - return m.Size() -} -func (m *CidList) XXX_DiscardUnknown() { - xxx_messageInfo_CidList.DiscardUnknown(m) -} - -var xxx_messageInfo_CidList proto.InternalMessageInfo - -func (m *CidList) GetCids() [][]byte { - if m != nil { - return m.Cids - } - return nil -} - -func init() { - proto.RegisterType((*CidEntry)(nil), "fileIndexProto.CidEntry") - proto.RegisterType((*CidList)(nil), "fileIndexProto.CidList") -} - -func init() { - proto.RegisterFile("index/redisindex/indexproto/protos/index.proto", fileDescriptor_01af1a9166444478) -} - -var fileDescriptor_01af1a9166444478 = []byte{ - // 206 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0xcb, 0xcc, 0x4b, 0x49, - 0xad, 0xd0, 0x2f, 0x4a, 0x4d, 0xc9, 0x2c, 0x86, 0x30, 0xc1, 0x64, 0x41, 0x51, 0x7e, 0x49, 0xbe, - 0x3e, 0x98, 0x2c, 0x86, 0x88, 0xe8, 0x81, 0x39, 0x42, 0x7c, 0x69, 0x99, 0x39, 0xa9, 0x9e, 0x20, - 0x81, 0x00, 0x10, 0x5f, 0xa9, 0x88, 0x8b, 0xc3, 0x39, 0x33, 0xc5, 0x35, 0xaf, 0xa4, 0xa8, 0x52, - 0x48, 0x88, 0x8b, 0xa5, 0x38, 0xb3, 0x2a, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x25, 0x08, 0xcc, - 0x16, 0x92, 0xe3, 0xe2, 0x4a, 0x2e, 0x4a, 0x4d, 0x2c, 0x49, 0x0d, 0xc9, 0xcc, 0x4d, 0x95, 0x60, - 0x52, 0x60, 0xd4, 0x60, 0x0e, 0x42, 0x12, 0x01, 0xc9, 0x97, 0x16, 0xa4, 0xc0, 0xe4, 0x99, 0x21, - 0xf2, 0x08, 0x11, 0x90, 0x99, 0x45, 0xa9, 0x69, 0xc5, 0x12, 0x2c, 0x0a, 0x8c, 0x1a, 0xac, 0x41, - 0x60, 0xb6, 0x92, 0x2c, 0x17, 0xbb, 0x73, 0x66, 0x8a, 0x4f, 0x66, 0x71, 0x09, 0x48, 0x3a, 0x39, - 0x33, 0xa5, 0x58, 0x82, 0x51, 0x81, 0x59, 0x83, 0x27, 0x08, 0xcc, 0x76, 0x32, 0x3d, 0xf1, 0x48, - 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, - 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, 0x86, 0x28, 0x69, 0x3c, 0x9e, 0x4d, 0x62, 0x03, 0x53, 0xc6, - 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd6, 0xa3, 0x20, 0x4c, 0x12, 0x01, 0x00, 0x00, -} - -func (m *CidEntry) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CidEntry) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CidEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Refs != 0 { - i = encodeVarintIndex(dAtA, i, uint64(m.Refs)) - i-- - dAtA[i] = 0x20 - } - if m.UpdateTime != 0 { - i = encodeVarintIndex(dAtA, i, uint64(m.UpdateTime)) - i-- - dAtA[i] = 0x18 - } - if m.CreateTime != 0 { - i = encodeVarintIndex(dAtA, i, uint64(m.CreateTime)) - i-- - dAtA[i] = 0x10 - } - if m.Size_ != 0 { - i = encodeVarintIndex(dAtA, i, uint64(m.Size_)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *CidList) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CidList) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *CidList) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Cids) > 0 { - for iNdEx := len(m.Cids) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Cids[iNdEx]) - copy(dAtA[i:], m.Cids[iNdEx]) - i = encodeVarintIndex(dAtA, i, uint64(len(m.Cids[iNdEx]))) - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func encodeVarintIndex(dAtA []byte, offset int, v uint64) int { - offset -= sovIndex(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *CidEntry) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Size_ != 0 { - n += 1 + sovIndex(uint64(m.Size_)) - } - if m.CreateTime != 0 { - n += 1 + sovIndex(uint64(m.CreateTime)) - } - if m.UpdateTime != 0 { - n += 1 + sovIndex(uint64(m.UpdateTime)) - } - if m.Refs != 0 { - n += 1 + sovIndex(uint64(m.Refs)) - } - return n -} - -func (m *CidList) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Cids) > 0 { - for _, b := range m.Cids { - l = len(b) - n += 1 + l + sovIndex(uint64(l)) - } - } - return n -} - -func sovIndex(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozIndex(x uint64) (n int) { - return sovIndex(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *CidEntry) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CidEntry: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CidEntry: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) - } - m.Size_ = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Size_ |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field CreateTime", wireType) - } - m.CreateTime = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.CreateTime |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field UpdateTime", wireType) - } - m.UpdateTime = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.UpdateTime |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Refs", wireType) - } - m.Refs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Refs |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipIndex(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthIndex - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *CidList) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CidList: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CidList: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Cids", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowIndex - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthIndex - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthIndex - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Cids = append(m.Cids, make([]byte, postIndex-iNdEx)) - copy(m.Cids[len(m.Cids)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipIndex(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthIndex - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipIndex(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowIndex - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowIndex - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowIndex - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthIndex - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupIndex - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthIndex - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthIndex = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowIndex = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupIndex = fmt.Errorf("proto: unexpected end of group") -) diff --git a/index/redisindex/indexproto/protos/index.proto b/index/redisindex/indexproto/protos/index.proto deleted file mode 100644 index f524a08f..00000000 --- a/index/redisindex/indexproto/protos/index.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; -package fileIndexProto; - -option go_package = "index/redisindex/indexproto"; - -message CidEntry { - uint64 size = 1; - int64 createTime = 2; - int64 updateTime = 3; - int32 refs = 4; -} - -message CidList { - repeated bytes cids = 1; -} \ No newline at end of file diff --git a/index/redisindex/redisindex.go b/index/redisindex/redisindex.go deleted file mode 100644 index 735bde5d..00000000 --- a/index/redisindex/redisindex.go +++ /dev/null @@ -1,372 +0,0 @@ -package redisindex - -import ( - "context" - "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/index/redisindex/indexproto" - "github.com/anyproto/any-sync-filenode/redisprovider" - "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/app/logger" - "github.com/go-redsync/redsync/v4" - "github.com/go-redsync/redsync/v4/redis/goredis/v9" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - "github.com/redis/go-redis/v9" - "strconv" - "strings" - "time" -) - -/* - Redis db structure: - CIDS: - c:{cid}: proto(Entry) - - STORES: - s:{storeKey}: map - f:{fileId} -> snappy(proto(CidList)) - {cid} -> int(refCount) - size -> int(summarySize) - -*/ - -const CName = "filenode.redisindex" - -var log = logger.NewNamed(CName) - -const ( - storeSizeKey = "size" -) - -func New() index.Index { - return new(redisIndex) -} - -type CidInfo struct { - Cid cid.Cid - *indexproto.CidEntry -} - -type redisIndex struct { - cl redis.UniversalClient - redsync *redsync.Redsync -} - -func (r *redisIndex) Init(a *app.App) (err error) { - r.cl = a.MustComponent(redisprovider.CName).(redisprovider.RedisProvider).Redis() - r.redsync = redsync.New(goredis.NewPool(r.cl)) - return nil -} - -func (r *redisIndex) Name() (name string) { - return CName -} - -func (r *redisIndex) Exists(ctx context.Context, k cid.Cid) (exists bool, err error) { - res, err := r.cl.Exists(ctx, cidKey(k.String())).Result() - if err != nil { - return - } - return res != 0, nil -} - -func (r *redisIndex) IsAllExists(ctx context.Context, cids []cid.Cid) (exists bool, err error) { - for _, c := range cids { - if ex, e := r.Exists(ctx, c); e != nil { - return false, e - } else if !ex { - return false, nil - } - } - return true, nil -} - -func (r *redisIndex) GetNonExistentBlocks(ctx context.Context, bs []blocks.Block) (nonExistent []blocks.Block, err error) { - nonExistent = make([]blocks.Block, 0, len(bs)) - for _, b := range bs { - if ex, e := r.Exists(ctx, b.Cid()); e != nil { - return nil, e - } else if !ex { - nonExistent = append(nonExistent, b) - } - } - return -} - -func (r *redisIndex) Bind(ctx context.Context, key, fileId string, bs []blocks.Block) error { - bop := bindOp{ - sk: storageKey(key), - fk: fileIdKey(fileId), - ri: r, - } - newCids, err := bop.Bind(ctx, r.cidInfoByBlocks(bs)) - if err != nil { - return err - } - return r.cidsAddRef(ctx, newCids) -} - -func (r *redisIndex) BindCids(ctx context.Context, key, fileId string, ks []cid.Cid) error { - cids, err := r.cidInfoByKeys(ctx, ks) - if err != nil { - return err - } - if len(cids) != len(ks) { - return index.ErrCidsNotExist - } - bop := bindOp{ - sk: storageKey(key), - fk: fileIdKey(fileId), - ri: r, - } - newCids, err := bop.Bind(ctx, cids) - if err != nil { - return err - } - return r.cidsAddRef(ctx, newCids) -} - -func (r *redisIndex) UnBind(ctx context.Context, spaceId, fileId string) (err error) { - uop := unbindOp{ - sk: storageKey(spaceId), - fk: fileIdKey(fileId), - ri: r, - } - removedCids, err := uop.Unbind(ctx) - if err != nil { - return - } - return r.cidsRemoveRef(ctx, removedCids) -} - -func (r *redisIndex) ExistsInStorage(ctx context.Context, spaceId string, ks []cid.Cid) (exists []cid.Cid, err error) { - var sk = storageKey(spaceId) - cidKeys := make([]string, len(ks)) - for i, k := range ks { - cidKeys[i] = k.String() - } - result, err := r.cl.HMGet(ctx, sk, cidKeys...).Result() - if err != nil { - return - } - exists = make([]cid.Cid, 0, len(ks)) - for i, v := range result { - if v != nil { - exists = append(exists, ks[i]) - } - } - return -} - -func (r *redisIndex) StorageSize(ctx context.Context, key string) (size uint64, err error) { - result, err := r.cl.HGet(ctx, storageKey(key), storeSizeKey).Result() - if err != nil { - if err == redis.Nil { - err = nil - } - return - } - return strconv.ParseUint(result, 10, 64) -} - -func (r *redisIndex) StorageInfo(ctx context.Context, key string) (info index.StorageInfo, err error) { - res, err := r.cl.HKeys(ctx, storageKey(key)).Result() - if err != nil { - if err == redis.Nil { - err = nil - } - return - } - for _, r := range res { - if strings.HasPrefix(r, "f:") { - info.FileCount++ - } else if r != storeSizeKey { - info.CidCount++ - } - } - info.Key = key - return -} - -func (r *redisIndex) FileInfo(ctx context.Context, key, fileId string) (info index.FileInfo, err error) { - fcl, err := r.newFileCidList(ctx, storageKey(key), fileIdKey(fileId)) - if err != nil { - return - } - defer fcl.Unlock() - cids, err := r.cidInfoByByteKeys(ctx, fcl.Cids) - if err != nil { - return - } - for _, c := range cids { - info.CidCount++ - info.BytesUsage += c.Size_ - } - return -} - -func (r *redisIndex) MoveStorage(ctx context.Context, fromKey, toKey string) (err error) { - ok, err := r.cl.RenameNX(ctx, storageKey(fromKey), storageKey(toKey)).Result() - if err != nil { - if err.Error() == "ERR no such key" { - return index.ErrStorageNotFound - } - return err - } - if !ok { - ex, err := r.cl.Exists(ctx, storageKey(toKey)).Result() - if err != nil { - return err - } - if ex > 0 { - return index.ErrTargetStorageExists - } - } - return -} - -func (r *redisIndex) cidsAddRef(ctx context.Context, cids []CidInfo) error { - now := time.Now() - for _, c := range cids { - ck := cidKey(c.Cid.String()) - res, err := r.cl.Get(ctx, ck).Result() - if err == redis.Nil { - err = nil - } - if err != nil { - return err - } - entry := &indexproto.CidEntry{} - if len(res) != 0 { - if err = entry.Unmarshal([]byte(res)); err != nil { - return err - } - } - if entry.CreateTime == 0 { - entry.CreateTime = now.Unix() - } - entry.UpdateTime = now.Unix() - entry.Refs++ - entry.Size_ = c.Size_ - - data, _ := entry.Marshal() - if err = r.cl.Set(ctx, ck, data, 0).Err(); err != nil { - return err - } - } - return nil -} - -func (r *redisIndex) cidsRemoveRef(ctx context.Context, cids []CidInfo) error { - now := time.Now() - for _, c := range cids { - ck := cidKey(c.Cid.String()) - res, err := r.cl.Get(ctx, ck).Result() - if err == redis.Nil { - continue - } - if err != nil { - return err - } - if len(res) == 0 { - continue - } - entry := &indexproto.CidEntry{} - if err = entry.Unmarshal([]byte(res)); err != nil { - return err - } - entry.UpdateTime = now.Unix() - entry.Refs-- - - // TODO: syncpool - data, _ := entry.Marshal() - if err = r.cl.Set(ctx, ck, data, 0).Err(); err != nil { - return err - } - } - return nil -} - -func (r *redisIndex) Lock(ctx context.Context, ks []cid.Cid) (unlock func(), err error) { - var lockers = make([]*redsync.Mutex, 0, len(ks)) - unlock = func() { - for _, l := range lockers { - _, _ = l.Unlock() - } - } - for _, k := range ks { - l := r.redsync.NewMutex("_lock:" + k.String()) - if err = l.LockContext(ctx); err != nil { - unlock() - return nil, err - } - lockers = append(lockers, l) - } - return -} - -func (r *redisIndex) AddBlocks(ctx context.Context, bs []blocks.Block) error { - cids := r.cidInfoByBlocks(bs) - if err := r.cidsAddRef(ctx, cids); err != nil { - return err - } - return r.cidsRemoveRef(ctx, cids) -} - -func (r *redisIndex) cidInfoByBlocks(bs []blocks.Block) (info []CidInfo) { - info = make([]CidInfo, len(bs)) - for i := range bs { - info[i] = CidInfo{ - Cid: bs[i].Cid(), - CidEntry: &indexproto.CidEntry{ - Size_: uint64(len(bs[i].RawData())), - }, - } - } - return -} - -func (r *redisIndex) cidInfoByKeys(ctx context.Context, ks []cid.Cid) (info []CidInfo, err error) { - info = make([]CidInfo, 0, len(ks)) - for _, c := range ks { - var res string - res, err = r.cl.Get(ctx, cidKey(c.String())).Result() - if err == redis.Nil { - continue - } - if err != nil { - return - } - entry := &indexproto.CidEntry{} - if err = entry.Unmarshal([]byte(res)); err != nil { - return - } - info = append(info, CidInfo{ - Cid: c, - CidEntry: entry, - }) - } - return -} - -func (r *redisIndex) cidInfoByByteKeys(ctx context.Context, ks [][]byte) (info []CidInfo, err error) { - var cids = make([]cid.Cid, 0, len(ks)) - for _, k := range ks { - if c, e := cid.Cast(k); e == nil { - cids = append(cids, c) - } - } - return r.cidInfoByKeys(ctx, cids) -} - -func storageKey(storeKey string) string { - return "s:" + storeKey -} - -func fileIdKey(fileId string) string { - return "f:" + fileId -} - -func cidKey(k string) string { - return "c:" + k -} diff --git a/index/redisindex/redisindex_test.go b/index/redisindex/redisindex_test.go deleted file mode 100644 index 902f53d8..00000000 --- a/index/redisindex/redisindex_test.go +++ /dev/null @@ -1,220 +0,0 @@ -package redisindex - -import ( - "context" - "github.com/anyproto/any-sync-filenode/index" - "github.com/anyproto/any-sync-filenode/redisprovider/testredisprovider" - "github.com/anyproto/any-sync-filenode/testutil" - "github.com/anyproto/any-sync/app" - blocks "github.com/ipfs/go-block-format" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "math/rand" - "strings" - "testing" - "time" -) - -var ctx = context.Background() - -func TestRedisIndex_Exists(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - storeKey := testutil.NewRandSpaceId() - var bs = make([]blocks.Block, 1) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(256 * 1024)) - } - require.NoError(t, fx.Bind(ctx, storeKey, "", bs)) - ex, err := fx.Exists(ctx, bs[0].Cid()) - require.NoError(t, err) - assert.True(t, ex) -} - -func TestRedisIndex_ExistsInSpace(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - storeKey := testutil.NewRandSpaceId() - var bs = make([]blocks.Block, 2) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(256 * 1024)) - } - require.NoError(t, fx.Bind(ctx, storeKey, testutil.NewRandCid().String(), bs[:1])) - ex, err := fx.ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)) - require.NoError(t, err) - assert.Len(t, ex, 1) -} - -func TestRedisIndex_IsAllExists(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - storeKey := testutil.NewRandSpaceId() - fileId := testutil.NewRandCid().String() - var bs = make([]blocks.Block, 2) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(256 * 1024)) - } - require.NoError(t, fx.Bind(ctx, storeKey, fileId, bs[:1])) - keys := testutil.BlocksToKeys(bs) - exists, err := fx.IsAllExists(ctx, keys) - require.NoError(t, err) - assert.False(t, exists) - exists, err = fx.IsAllExists(ctx, keys[:1]) - require.NoError(t, err) - assert.True(t, exists) -} - -func TestRedisIndex_GetNonExistentBlocks(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - storeKey := testutil.NewRandSpaceId() - fileId := testutil.NewRandCid().String() - var bs = make([]blocks.Block, 2) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(256 * 1024)) - } - require.NoError(t, fx.Bind(ctx, storeKey, fileId, bs[:1])) - - nonExistent, err := fx.GetNonExistentBlocks(ctx, bs) - require.NoError(t, err) - require.Len(t, nonExistent, 1) - assert.Equal(t, bs[1:], nonExistent) -} - -func TestRedisIndex_SpaceSize(t *testing.T) { - t.Run("space not found", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - storeKey := testutil.NewRandSpaceId() - size, err := fx.StorageSize(ctx, storeKey) - require.NoError(t, err) - assert.Empty(t, size) - }) - t.Run("success", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - var storeKey = testutil.NewRandSpaceId() - var expectedSize int - fileId := testutil.NewRandCid().String() - var bs = make([]blocks.Block, 2) - for i := range bs { - bs[i] = testutil.NewRandBlock(1024) - expectedSize += 1024 - } - require.NoError(t, fx.AddBlocks(ctx, bs)) - require.NoError(t, fx.Bind(ctx, storeKey, fileId, bs)) - - size, err := fx.StorageSize(ctx, storeKey) - require.NoError(t, err) - assert.Equal(t, expectedSize, int(size)) - }) -} - -func TestRedisIndex_Lock(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - - var bs = make([]blocks.Block, 3) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(1024)) - } - - unlock, err := fx.Lock(ctx, testutil.BlocksToKeys(bs[1:])) - require.NoError(t, err) - tCtx, cancel := context.WithTimeout(ctx, time.Second/2) - defer cancel() - _, err = fx.Lock(tCtx, testutil.BlocksToKeys(bs)) - require.Error(t, err) - unlock() -} - -func TestRedisIndex_MoveStorage(t *testing.T) { - const ( - oldKey = "oldKey" - newKey = "newKey" - ) - t.Run("success", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1))) - require.NoError(t, fx.MoveStorage(ctx, oldKey, newKey)) - }) - t.Run("err storage not found", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrStorageNotFound.Error()) - }) - t.Run("err taget exists", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1))) - require.NoError(t, fx.Bind(ctx, newKey, "fid", testutil.NewRandBlocks(1))) - assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrTargetStorageExists.Error()) - }) -} - -func Test100KCids(t *testing.T) { - t.Skip() - fx := newFixture(t) - defer fx.Finish(t) - for i := 0; i < 10; i++ { - st := time.Now() - var bs = make([]blocks.Block, 10000) - for n := range bs { - bs[n] = testutil.NewRandBlock(rand.Intn(256)) - } - storeKey := testutil.NewRandSpaceId() - fileId := testutil.NewRandCid().String() - require.NoError(t, fx.Bind(ctx, storeKey, fileId, bs)) - t.Logf("bound %d cid for a %v", len(bs), time.Since(st)) - st = time.Now() - sz, err := fx.StorageSize(ctx, storeKey) - require.NoError(t, err) - t.Logf("space size is %d, dur: %v", sz, time.Since(st)) - } - info, err := fx.cl.Info(ctx, "memory").Result() - require.NoError(t, err) - infoS := strings.Split(info, "\n") - for _, i := range infoS { - if strings.HasPrefix(i, "used_memory_human") { - t.Log(i) - } - } -} - -func TestRedisIndex_AddBlocks(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - - var bs = make([]blocks.Block, 3) - for i := range bs { - bs[i] = testutil.NewRandBlock(rand.Intn(1024)) - } - - require.NoError(t, fx.AddBlocks(ctx, bs)) - - for _, b := range bs { - ex, err := fx.Exists(ctx, b.Cid()) - require.NoError(t, err) - assert.True(t, ex) - } -} - -func newFixture(t require.TestingT) (fx *fixture) { - fx = &fixture{ - redisIndex: New().(*redisIndex), - a: new(app.App), - } - fx.a.Register(testredisprovider.NewTestRedisProvider()).Register(fx.redisIndex) - require.NoError(t, fx.a.Start(ctx)) - return -} - -type fixture struct { - *redisIndex - a *app.App -} - -func (fx *fixture) Finish(t require.TestingT) { - require.NoError(t, fx.a.Close(ctx)) -} diff --git a/index/redisindex/unbind.go b/index/redisindex/unbind.go deleted file mode 100644 index 74ba2305..00000000 --- a/index/redisindex/unbind.go +++ /dev/null @@ -1,86 +0,0 @@ -package redisindex - -import ( - "context" - "github.com/ipfs/go-cid" - "github.com/redis/go-redis/v9" - "go.uber.org/zap" -) - -type unbindOp struct { - sk string - fk string - ri *redisIndex -} - -func (op *unbindOp) Unbind(ctx context.Context) (removedCids []CidInfo, err error) { - // fetch cids list - fCids, err := op.ri.newFileCidList(ctx, op.sk, op.fk) - if err != nil { - return nil, err - } - defer fCids.Unlock() - - // no cids by file, probably it doesn't exist - if len(fCids.Cids) == 0 { - return nil, nil - } - - // fetch cids info - cids, err := op.ri.cidInfoByByteKeys(ctx, fCids.Cids) - if err != nil { - return nil, err - } - // additional check - if len(cids) != len(fCids.Cids) { - log.Warn("can't fetch all file cids") - } - fCids.Cids = nil - - // we need to lock cids here - cidsToLock := make([]cid.Cid, len(cids)) - for i, c := range cids { - cidsToLock[i] = c.Cid - } - unlock, err := op.ri.Lock(ctx, cidsToLock) - if err != nil { - return nil, err - } - defer unlock() - - // make a list of results, data will be available after executing of pipeline - var execResults = make([]*redis.IntCmd, len(cids)) - // do updates in one pipeline - _, err = op.ri.cl.TxPipelined(ctx, func(pipe redis.Pipeliner) error { - for i, c := range cids { - execResults[i] = pipe.HIncrBy(ctx, op.sk, c.Cid.String(), -1) - if err = execResults[i].Err(); err != nil { - return err - } - } - return fCids.Save(ctx, pipe) - }) - - // check for cids that were removed from a space - // remove cids without reference - // make a list with removed cids - // calculate space size decrease - var spaceDecreaseSize uint64 - for i, res := range execResults { - if counter, err := res.Result(); err == nil && counter <= 0 { - if e := op.ri.cl.HDel(ctx, op.sk, cids[i].Cid.String()).Err(); e != nil { - log.Warn("can't remove cid from a space", zap.Error(e)) - } - removedCids = append(removedCids, cids[i]) - spaceDecreaseSize += cids[i].Size_ - } - } - - // increment space size - if spaceDecreaseSize > 0 { - if err = op.ri.cl.HIncrBy(ctx, op.sk, storeSizeKey, -int64(spaceDecreaseSize)).Err(); err != nil { - return nil, err - } - } - return -} diff --git a/index/redisindex/unbind_test.go b/index/redisindex/unbind_test.go deleted file mode 100644 index 7ea65352..00000000 --- a/index/redisindex/unbind_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package redisindex - -import ( - "github.com/anyproto/any-sync-filenode/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" -) - -func TestRedisIndex_UnBind(t *testing.T) { - t.Run("unbind non existent", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - spaceId := testutil.NewRandSpaceId() - require.NoError(t, fx.UnBind(ctx, spaceId, testutil.NewRandCid().String())) - }) - t.Run("unbind single", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - spaceId := testutil.NewRandSpaceId() - bs := testutil.NewRandBlocks(5) - fileId := testutil.NewRandCid().String() - require.NoError(t, fx.Bind(ctx, spaceId, fileId, bs)) - - require.NoError(t, fx.UnBind(ctx, spaceId, fileId)) - size, err := fx.StorageSize(ctx, spaceId) - require.NoError(t, err) - assert.Empty(t, size) - - var cidsB [][]byte - for _, b := range bs { - cidsB = append(cidsB, b.Cid().Bytes()) - } - - res, err := fx.cidInfoByByteKeys(ctx, cidsB) - require.NoError(t, err) - for _, r := range res { - assert.Equal(t, int32(0), r.Refs) - } - }) - t.Run("unbind two files", func(t *testing.T) { - fx := newFixture(t) - defer fx.Finish(t) - spaceId := testutil.NewRandSpaceId() - bs := testutil.NewRandBlocks(5) - fileId1 := testutil.NewRandCid().String() - fileId2 := testutil.NewRandCid().String() - require.NoError(t, fx.Bind(ctx, spaceId, fileId1, bs)) - require.NoError(t, fx.Bind(ctx, spaceId, fileId2, bs[:3])) - - si, err := fx.StorageInfo(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, 2, si.FileCount) - assert.Equal(t, len(bs), si.CidCount) - - require.NoError(t, fx.UnBind(ctx, spaceId, fileId1)) - si, err = fx.StorageInfo(ctx, spaceId) - require.NoError(t, err) - assert.Equal(t, 1, si.FileCount) - assert.Equal(t, 3, si.CidCount) - }) -} diff --git a/index/testdata/oldspace.zip b/index/testdata/oldspace.zip new file mode 100644 index 00000000..193796d1 Binary files /dev/null and b/index/testdata/oldspace.zip differ diff --git a/index/unbind.go b/index/unbind.go new file mode 100644 index 00000000..9e857510 --- /dev/null +++ b/index/unbind.go @@ -0,0 +1,146 @@ +package index + +import ( + "context" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +func (ri *redisIndex) FileUnbind(ctx context.Context, key Key, fileIds ...string) (err error) { + _, gRelease, err := ri.AcquireKey(ctx, groupKey(key)) + if err != nil { + return + } + defer gRelease() + _, sRelease, err := ri.AcquireKey(ctx, spaceKey(key)) + if err != nil { + return + } + defer sRelease() + for _, fileId := range fileIds { + if err = ri.fileUnbind(ctx, key, fileId); err != nil { + return + } + } + return +} + +func (ri *redisIndex) fileUnbind(ctx context.Context, key Key, fileId string) (err error) { + var ( + sk = spaceKey(key) + gk = groupKey(key) + ) + // get file entry + fileInfo, isNewFile, err := ri.getFileEntry(ctx, key, fileId) + if err != nil { + return + } + if isNewFile { + // means file doesn't exist + return nil + } + + // fetch cids + cids, err := ri.CidEntriesByString(ctx, fileInfo.Cids) + if err != nil { + return err + } + defer cids.Release() + + // fetch cid refs in one pipeline + var ( + groupCidRefs = make([]*redis.StringCmd, len(cids.entries)) + spaceCidRefs = make([]*redis.StringCmd, len(cids.entries)) + ) + _, err = ri.cl.Pipelined(ctx, func(pipe redis.Pipeliner) error { + for i, c := range cids.entries { + groupCidRefs[i] = pipe.HGet(ctx, gk, cidKey(c.Cid)) + spaceCidRefs[i] = pipe.HGet(ctx, sk, cidKey(c.Cid)) + } + return nil + }) + if err != nil { + return + } + + // load group and space info + spaceInfo, err := ri.getSpaceEntry(ctx, key) + if err != nil { + return + } + groupInfo, err := ri.getGroupEntry(ctx, key) + if err != nil { + return + } + + // update info and calculate changes + var ( + groupRemoveKeys = make([]string, 0, len(cids.entries)) + spaceRemoveKeys = make([]string, 0, len(cids.entries)) + groupDecrKeys = make([]string, 0, len(cids.entries)) + spaceDecrKeys = make([]string, 0, len(cids.entries)) + affectedCidIdx = make([]int, 0, len(cids.entries)) + ) + + spaceInfo.FileCount-- + for i, c := range cids.entries { + res, err := groupCidRefs[i].Result() + if err != nil { + return err + } + ck := cidKey(c.Cid) + if res == "1" { + groupRemoveKeys = append(groupRemoveKeys, ck) + groupInfo.Size_ -= c.Size_ + groupInfo.CidCount-- + affectedCidIdx = append(affectedCidIdx, i) + } else { + groupDecrKeys = append(groupDecrKeys, ck) + } + res, err = spaceCidRefs[i].Result() + if err != nil { + return err + } + if res == "1" { + spaceRemoveKeys = append(spaceRemoveKeys, ck) + spaceInfo.Size_ -= c.Size_ + spaceInfo.CidCount-- + } else { + spaceDecrKeys = append(spaceDecrKeys, ck) + } + } + + // do updates in one tx + _, err = ri.cl.TxPipelined(ctx, func(tx redis.Pipeliner) error { + tx.HDel(ctx, sk, fileKey(fileId)) + if len(spaceRemoveKeys) != 0 { + tx.HDel(ctx, sk, spaceRemoveKeys...) + } + if len(groupRemoveKeys) != 0 { + tx.HDel(ctx, gk, groupRemoveKeys...) + } + if len(spaceDecrKeys) != 0 { + for _, k := range spaceDecrKeys { + tx.HIncrBy(ctx, sk, k, -1) + } + } + if len(groupDecrKeys) != 0 { + for _, k := range groupDecrKeys { + tx.HIncrBy(ctx, gk, k, -1) + } + } + spaceInfo.Save(ctx, key, tx) + groupInfo.Save(ctx, key, tx) + return nil + }) + + // update cids + for _, idx := range affectedCidIdx { + cids.entries[idx].Refs-- + if saveErr := cids.entries[idx].Save(ctx, ri.cl); saveErr != nil { + log.WarnCtx(ctx, "unable to save cid info", zap.Error(saveErr), zap.String("cid", cids.entries[idx].Cid.String())) + } + } + return +} diff --git a/index/unbind_test.go b/index/unbind_test.go new file mode 100644 index 00000000..5fd20031 --- /dev/null +++ b/index/unbind_test.go @@ -0,0 +1,93 @@ +package index + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync-filenode/testutil" +) + +func TestRedisIndex_UnBind(t *testing.T) { + t.Run("unbind non existent", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + key := newRandKey() + require.NoError(t, fx.FileUnbind(ctx, key, testutil.NewRandCid().String())) + }) + t.Run("unbind single", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + key := newRandKey() + bs := testutil.NewRandBlocks(5) + + require.NoError(t, fx.BlocksAdd(ctx, bs)) + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + + fileId := testutil.NewRandCid().String() + require.NoError(t, fx.FileBind(ctx, key, fileId, cids)) + cids.Release() + + require.NoError(t, fx.FileUnbind(ctx, key, fileId)) + + groupInfo, err := fx.GroupInfo(ctx, key.GroupId) + require.NoError(t, err) + assert.Empty(t, groupInfo.CidsCount) + assert.Empty(t, groupInfo.BytesUsage) + spaceInfo, err := fx.SpaceInfo(ctx, key) + require.NoError(t, err) + assert.Empty(t, spaceInfo.FileCount) + assert.Empty(t, spaceInfo.BytesUsage) + + cids, err = fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer cids.Release() + for _, r := range cids.entries { + assert.Equal(t, int32(0), r.Refs) + } + }) + t.Run("unbind intersection file", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + key := newRandKey() + bs := testutil.NewRandBlocks(5) + var file1Size = uint64(len(bs[0].RawData()) + len(bs[1].RawData())) + require.NoError(t, fx.BlocksAdd(ctx, bs)) + cids, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + cids.Release() + + // bind to files with intersected cids + fileId1 := testutil.NewRandCid().String() + fileId2 := testutil.NewRandCid().String() + + cids1, err := fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, key, fileId1, cids1)) + cids1.Release() + + cids2, err := fx.CidEntriesByBlocks(ctx, bs[:2]) + require.NoError(t, err) + require.NoError(t, fx.FileBind(ctx, key, fileId2, cids2)) + cids2.Release() + + // remove file1 + require.NoError(t, fx.FileUnbind(ctx, key, fileId1)) + + groupInfo, err := fx.GroupInfo(ctx, key.GroupId) + require.NoError(t, err) + assert.Equal(t, uint64(2), groupInfo.CidsCount) + assert.Equal(t, file1Size, groupInfo.BytesUsage) + spaceInfo, err := fx.SpaceInfo(ctx, key) + require.NoError(t, err) + assert.Equal(t, uint32(1), spaceInfo.FileCount) + assert.Equal(t, file1Size, spaceInfo.BytesUsage) + + cids, err = fx.CidEntriesByBlocks(ctx, bs) + require.NoError(t, err) + defer cids.Release() + + }) +} diff --git a/limit/mock_limit/mock_limit.go b/limit/mock_limit/mock_limit.go index 59f9585b..982266ab 100644 --- a/limit/mock_limit/mock_limit.go +++ b/limit/mock_limit/mock_limit.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/any-sync-filenode/limit (interfaces: Limit) - +// +// Generated by this command: +// +// mockgen -destination mock_limit/mock_limit.go github.com/anyproto/any-sync-filenode/limit Limit +// // Package mock_limit is a generated GoMock package. package mock_limit @@ -46,7 +50,7 @@ func (m *MockLimit) Check(arg0 context.Context, arg1 string) (uint64, string, er } // Check indicates an expected call of Check. -func (mr *MockLimitMockRecorder) Check(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockLimitMockRecorder) Check(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockLimit)(nil).Check), arg0, arg1) } @@ -60,7 +64,7 @@ func (m *MockLimit) Close(arg0 context.Context) error { } // Close indicates an expected call of Close. -func (mr *MockLimitMockRecorder) Close(arg0 interface{}) *gomock.Call { +func (mr *MockLimitMockRecorder) Close(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockLimit)(nil).Close), arg0) } @@ -74,7 +78,7 @@ func (m *MockLimit) Init(arg0 *app.App) error { } // Init indicates an expected call of Init. -func (mr *MockLimitMockRecorder) Init(arg0 interface{}) *gomock.Call { +func (mr *MockLimitMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockLimit)(nil).Init), arg0) } @@ -102,7 +106,7 @@ func (m *MockLimit) Run(arg0 context.Context) error { } // Run indicates an expected call of Run. -func (mr *MockLimitMockRecorder) Run(arg0 interface{}) *gomock.Call { +func (mr *MockLimitMockRecorder) Run(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockLimit)(nil).Run), arg0) } diff --git a/redisprovider/redisprovider.go b/redisprovider/redisprovider.go index 2d74beca..e79529cd 100644 --- a/redisprovider/redisprovider.go +++ b/redisprovider/redisprovider.go @@ -2,6 +2,7 @@ package redisprovider import ( "context" + "github.com/anyproto/any-sync/app" "github.com/redis/go-redis/v9" ) @@ -48,7 +49,13 @@ func (r *redisProvider) Name() (name string) { } func (r *redisProvider) Run(ctx context.Context) (err error) { - return r.redis.Ping(ctx).Err() + if err = r.redis.Ping(ctx).Err(); err != nil { + return + } + if err = r.redis.BFAdd(ctx, "_test_bf", 1).Err(); err != nil { + return + } + return r.redis.Del(ctx, "_test_bf").Err() } func (r *redisProvider) Redis() redis.UniversalClient { diff --git a/store/filedevstore/filedevstore.go b/store/filedevstore/filedevstore.go index 45482276..dcadb762 100644 --- a/store/filedevstore/filedevstore.go +++ b/store/filedevstore/filedevstore.go @@ -3,15 +3,17 @@ package filedevstore import ( "context" "fmt" - "github.com/anyproto/any-sync-filenode/config" - "github.com/anyproto/any-sync-filenode/store" + "os" + "path/filepath" + "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/commonfile/fileblockstore" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "os" - "path/filepath" + + "github.com/anyproto/any-sync-filenode/config" + "github.com/anyproto/any-sync-filenode/store" ) const CName = fileblockstore.CName @@ -96,3 +98,11 @@ func (s *fsstore) DeleteMany(ctx context.Context, toDelete []cid.Cid) error { } return nil } + +func (s *fsstore) IndexGet(ctx context.Context, key string) (value []byte, err error) { + return os.ReadFile(filepath.Join(s.path, key)) +} + +func (s *fsstore) IndexPut(ctx context.Context, key string, value []byte) (err error) { + return os.WriteFile(filepath.Join(s.path, key), value, 0777) +} diff --git a/store/mock_store/mock_store.go b/store/mock_store/mock_store.go index 1357d2f1..07c45c03 100644 --- a/store/mock_store/mock_store.go +++ b/store/mock_store/mock_store.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/any-sync-filenode/store (interfaces: Store) - +// +// Generated by this command: +// +// mockgen -destination mock_store/mock_store.go github.com/anyproto/any-sync-filenode/store Store +// // Package mock_store is a generated GoMock package. package mock_store @@ -46,7 +50,7 @@ func (m *MockStore) Add(arg0 context.Context, arg1 []blocks.Block) error { } // Add indicates an expected call of Add. -func (mr *MockStoreMockRecorder) Add(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) Add(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockStore)(nil).Add), arg0, arg1) } @@ -60,7 +64,7 @@ func (m *MockStore) Delete(arg0 context.Context, arg1 cid.Cid) error { } // Delete indicates an expected call of Delete. -func (mr *MockStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) Delete(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), arg0, arg1) } @@ -74,7 +78,7 @@ func (m *MockStore) DeleteMany(arg0 context.Context, arg1 []cid.Cid) error { } // DeleteMany indicates an expected call of DeleteMany. -func (mr *MockStoreMockRecorder) DeleteMany(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) DeleteMany(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMany", reflect.TypeOf((*MockStore)(nil).DeleteMany), arg0, arg1) } @@ -89,7 +93,7 @@ func (m *MockStore) Get(arg0 context.Context, arg1 cid.Cid) (blocks.Block, error } // Get indicates an expected call of Get. -func (mr *MockStoreMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) Get(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStore)(nil).Get), arg0, arg1) } @@ -103,11 +107,40 @@ func (m *MockStore) GetMany(arg0 context.Context, arg1 []cid.Cid) <-chan blocks. } // GetMany indicates an expected call of GetMany. -func (mr *MockStoreMockRecorder) GetMany(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) GetMany(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMany", reflect.TypeOf((*MockStore)(nil).GetMany), arg0, arg1) } +// IndexGet mocks base method. +func (m *MockStore) IndexGet(arg0 context.Context, arg1 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexGet", arg0, arg1) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IndexGet indicates an expected call of IndexGet. +func (mr *MockStoreMockRecorder) IndexGet(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexGet", reflect.TypeOf((*MockStore)(nil).IndexGet), arg0, arg1) +} + +// IndexPut mocks base method. +func (m *MockStore) IndexPut(arg0 context.Context, arg1 string, arg2 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexPut", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// IndexPut indicates an expected call of IndexPut. +func (mr *MockStoreMockRecorder) IndexPut(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexPut", reflect.TypeOf((*MockStore)(nil).IndexPut), arg0, arg1, arg2) +} + // Init mocks base method. func (m *MockStore) Init(arg0 *app.App) error { m.ctrl.T.Helper() @@ -117,7 +150,7 @@ func (m *MockStore) Init(arg0 *app.App) error { } // Init indicates an expected call of Init. -func (mr *MockStoreMockRecorder) Init(arg0 interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockStore)(nil).Init), arg0) } diff --git a/store/s3store/config.go b/store/s3store/config.go index 2138a0fa..8ecdb6d1 100644 --- a/store/s3store/config.go +++ b/store/s3store/config.go @@ -13,6 +13,7 @@ type Config struct { Profile string `yaml:"profile"` Region string `yaml:"region"` Bucket string `yaml:"bucket"` + IndexBucket string `yaml:"indexBucket"` Endpoint string `yaml:"endpoint"` MaxThreads int `yaml:"maxThreads"` Credentials Credentials `yaml:"credentials"` diff --git a/store/s3store/s3store.go b/store/s3store/s3store.go index 18c2c9a4..c12ddc04 100644 --- a/store/s3store/s3store.go +++ b/store/s3store/s3store.go @@ -5,10 +5,10 @@ import ( "context" "fmt" "io" + "strings" "sync" "time" - "github.com/anyproto/any-sync-filenode/store" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/commonfile/fileblockstore" @@ -19,6 +19,8 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "go.uber.org/zap" + + "github.com/anyproto/any-sync-filenode/store" ) const CName = fileblockstore.CName @@ -35,10 +37,11 @@ type S3Store interface { } type s3store struct { - bucket *string - client *s3.S3 - limiter chan struct{} - sess *session.Session + bucket *string + indexBucket *string + client *s3.S3 + limiter chan struct{} + sess *session.Session } func (s *s3store) Init(a *app.App) (err error) { @@ -49,6 +52,9 @@ func (s *s3store) Init(a *app.App) (err error) { if conf.Bucket == "" { return fmt.Errorf("s3 bucket is empty") } + if conf.IndexBucket == "" { + return fmt.Errorf("s3 index bucket is empty") + } if conf.MaxThreads <= 0 { conf.MaxThreads = 16 } @@ -81,6 +87,8 @@ func (s *s3store) Init(a *app.App) (err error) { return fmt.Errorf("failed to create session to s3: %v", err) } s.bucket = aws.String(conf.Bucket) + s.indexBucket = aws.String(conf.IndexBucket) + s.client = s3.New(s.sess) s.limiter = make(chan struct{}, conf.MaxThreads) return nil @@ -104,6 +112,9 @@ func (s *s3store) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { Key: aws.String(k.String()), }) if err != nil { + if strings.HasPrefix(err.Error(), s3.ErrCodeNoSuchKey) { + return nil, fileblockstore.ErrCIDNotFound + } return nil, err } defer obj.Body.Close() @@ -203,6 +214,31 @@ func (s *s3store) Delete(ctx context.Context, c cid.Cid) error { return err } +func (s *s3store) IndexGet(ctx context.Context, key string) (value []byte, err error) { + obj, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: s.indexBucket, + Key: aws.String(key), + }) + if err != nil { + if strings.HasPrefix(err.Error(), s3.ErrCodeNoSuchKey) { + // nil value means not found + return nil, nil + } + return nil, err + } + defer obj.Body.Close() + return io.ReadAll(obj.Body) +} + +func (s *s3store) IndexPut(ctx context.Context, key string, data []byte) (err error) { + _, err = s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{ + Key: aws.String(key), + Body: bytes.NewReader(data), + Bucket: s.indexBucket, + }) + return +} + func (s *s3store) Close(ctx context.Context) (err error) { return nil } diff --git a/store/s3store/s3store_test.go b/store/s3store/s3store_test.go index 75a44892..55c9694f 100644 --- a/store/s3store/s3store_test.go +++ b/store/s3store/s3store_test.go @@ -3,13 +3,14 @@ package s3store import ( "context" "fmt" + "testing" + "time" + "github.com/anyproto/any-sync/app" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "testing" - "time" ) var ctx = context.Background() @@ -57,8 +58,9 @@ func (c config) Name() string { return "config" } func (c config) GetS3Store() Config { return Config{ - Region: "eu-central-1", - Bucket: "anytype-test", - MaxThreads: 4, + Region: "eu-central-1", + Bucket: "anytype-test", + IndexBucket: "anytype-test", + MaxThreads: 4, } } diff --git a/store/store.go b/store/store.go index f50ca768..e780bf1e 100644 --- a/store/store.go +++ b/store/store.go @@ -3,6 +3,7 @@ package store import ( "context" + "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/commonfile/fileblockstore" "github.com/ipfs/go-cid" @@ -11,5 +12,8 @@ import ( type Store interface { fileblockstore.BlockStore DeleteMany(ctx context.Context, toDelete []cid.Cid) error + + IndexGet(ctx context.Context, key string) (value []byte, err error) + IndexPut(ctx context.Context, key string, value []byte) (err error) app.Component }