Skip to content

Commit

Permalink
move space to storageKey
Browse files Browse the repository at this point in the history
  • Loading branch information
cheggaaa committed Sep 15, 2023
1 parent b6a6ce1 commit 05f4728
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 3 deletions.
8 changes: 8 additions & 0 deletions filenode/filenode.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,14 @@ func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit boo
return
}

if storageKey != spaceId {
// try to move store to the new key
mErr := fn.index.MoveStorage(ctx, spaceId, storageKey)
if mErr != nil && mErr != index.ErrStorageNotFound && mErr != index.ErrTargetStorageExists {
return "", mErr
}
}

if checkLimit {
currentSize, e := fn.index.StorageSize(ctx, storageKey)
if e != nil {
Expand Down
5 changes: 5 additions & 0 deletions filenode/filenode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func TestFileNode_Add(t *testing.T) {
)

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(120), nil)
fx.index.EXPECT().Lock(ctx, []cid.Cid{b.Cid()}).Return(func() {}, nil)
fx.index.EXPECT().GetNonExistentBlocks(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil)
Expand All @@ -64,6 +65,7 @@ func TestFileNode_Add(t *testing.T) {
)

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(124), nil)

resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{
Expand Down Expand Up @@ -155,6 +157,7 @@ func TestFileNode_Check(t *testing.T) {
cids = append(cids, b.Cid().Bytes())
}
fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(100000), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil)
fx.index.EXPECT().Exists(ctx, bs[1].Cid()).Return(true, nil)
fx.index.EXPECT().Exists(ctx, bs[2].Cid()).Return(false, nil)
Expand Down Expand Up @@ -186,6 +189,7 @@ func TestFileNode_BlocksBind(t *testing.T) {
}

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(12), nil)
fx.index.EXPECT().Lock(ctx, cids).Return(func() {}, nil)
fx.index.EXPECT().BindCids(ctx, storeKey, fileId, cids)
Expand All @@ -210,6 +214,7 @@ func TestFileNode_FileInfo(t *testing.T) {
fileId2 = testutil.NewRandCid().String()
)
fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes().Return(uint64(100000), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1).Return(index.FileInfo{1, 1}, nil)
fx.index.EXPECT().FileInfo(ctx, storeKey, fileId2).Return(index.FileInfo{2, 2}, nil)

Expand Down
5 changes: 4 additions & 1 deletion index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import (
)

var (
ErrCidsNotExist = errors.New("cids not exist")
ErrCidsNotExist = errors.New("cids not exist")
ErrTargetStorageExists = errors.New("target storage exists")
ErrStorageNotFound = errors.New("storage not found")
)

type Index interface {
Expand All @@ -26,6 +28,7 @@ type Index interface {
StorageSize(ctx context.Context, key string) (size uint64, err error)
Lock(ctx context.Context, ks []cid.Cid) (unlock func(), err error)
AddBlocks(ctx context.Context, upload []blocks.Block) error
MoveStorage(ctx context.Context, fromKey, toKey string) error
app.Component
}

Expand Down
14 changes: 14 additions & 0 deletions index/mock_index/mock_index.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 22 additions & 2 deletions index/redisindex/redisindex.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,26 @@ func (r *redisIndex) FileInfo(ctx context.Context, key, fileId string) (info ind
return
}

func (r *redisIndex) MoveStorage(ctx context.Context, fromKey, toKey string) (err error) {
ok, err := r.cl.RenameNX(ctx, storageKey(fromKey), storageKey(toKey)).Result()
if err != nil {
if err.Error() == "ERR no such key" {
return index.ErrStorageNotFound
}
return err
}
if !ok {
ex, err := r.cl.Exists(ctx, storageKey(toKey)).Result()
if err != nil {
return err
}
if ex > 0 {
return index.ErrTargetStorageExists
}
}
return
}

func (r *redisIndex) cidsAddRef(ctx context.Context, cids []CidInfo) error {
now := time.Now()
for _, c := range cids {
Expand Down Expand Up @@ -339,8 +359,8 @@ func (r *redisIndex) cidInfoByByteKeys(ctx context.Context, ks [][]byte) (info [
return r.cidInfoByKeys(ctx, cids)
}

func storageKey(spaceId string) string {
return "s:" + spaceId
func storageKey(storeKey string) string {
return "s:" + storeKey
}

func fileIdKey(fileId string) string {
Expand Down
26 changes: 26 additions & 0 deletions index/redisindex/redisindex_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package redisindex

import (
"context"
"github.com/anyproto/any-sync-filenode/index"
"github.com/anyproto/any-sync-filenode/redisprovider/testredisprovider"
"github.com/anyproto/any-sync-filenode/testutil"
"github.com/anyproto/any-sync/app"
Expand Down Expand Up @@ -127,6 +128,31 @@ func TestRedisIndex_Lock(t *testing.T) {
unlock()
}

func TestRedisIndex_MoveStorage(t *testing.T) {
const (
oldKey = "oldKey"
newKey = "newKey"
)
t.Run("success", func(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1)))
require.NoError(t, fx.MoveStorage(ctx, oldKey, newKey))
})
t.Run("err storage not found", func(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrStorageNotFound.Error())
})
t.Run("err taget exists", func(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1)))
require.NoError(t, fx.Bind(ctx, newKey, "fid", testutil.NewRandBlocks(1)))
assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrTargetStorageExists.Error())
})
}

func Test100KCids(t *testing.T) {
t.Skip()
fx := newFixture(t)
Expand Down

0 comments on commit 05f4728

Please sign in to comment.