From 05f472803ed372662d14fb2673b182fe3d06489f Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 15 Sep 2023 13:31:06 +0200 Subject: [PATCH] move space to storageKey --- filenode/filenode.go | 8 ++++++++ filenode/filenode_test.go | 5 +++++ index/index.go | 5 ++++- index/mock_index/mock_index.go | 14 ++++++++++++++ index/redisindex/redisindex.go | 24 ++++++++++++++++++++++-- index/redisindex/redisindex_test.go | 26 ++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/filenode/filenode.go b/filenode/filenode.go index a4a0680..6e857b2 100644 --- a/filenode/filenode.go +++ b/filenode/filenode.go @@ -150,6 +150,14 @@ func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit boo return } + if storageKey != spaceId { + // try to move store to the new key + mErr := fn.index.MoveStorage(ctx, spaceId, storageKey) + if mErr != nil && mErr != index.ErrStorageNotFound && mErr != index.ErrTargetStorageExists { + return "", mErr + } + } + if checkLimit { currentSize, e := fn.index.StorageSize(ctx, storageKey) if e != nil { diff --git a/filenode/filenode_test.go b/filenode/filenode_test.go index dc51c18..d0ef4a8 100644 --- a/filenode/filenode_test.go +++ b/filenode/filenode_test.go @@ -38,6 +38,7 @@ func TestFileNode_Add(t *testing.T) { ) fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) + fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(120), nil) fx.index.EXPECT().Lock(ctx, []cid.Cid{b.Cid()}).Return(func() {}, nil) fx.index.EXPECT().GetNonExistentBlocks(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil) @@ -64,6 +65,7 @@ func TestFileNode_Add(t *testing.T) { ) fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) + fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(124), nil) resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{ @@ -155,6 +157,7 @@ func TestFileNode_Check(t *testing.T) { cids = append(cids, b.Cid().Bytes()) } fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(100000), storeKey, nil) + fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() fx.index.EXPECT().ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil) fx.index.EXPECT().Exists(ctx, bs[1].Cid()).Return(true, nil) fx.index.EXPECT().Exists(ctx, bs[2].Cid()).Return(false, nil) @@ -186,6 +189,7 @@ func TestFileNode_BlocksBind(t *testing.T) { } fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil) + fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(12), nil) fx.index.EXPECT().Lock(ctx, cids).Return(func() {}, nil) fx.index.EXPECT().BindCids(ctx, storeKey, fileId, cids) @@ -210,6 +214,7 @@ func TestFileNode_FileInfo(t *testing.T) { fileId2 = testutil.NewRandCid().String() ) fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes().Return(uint64(100000), storeKey, nil) + fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes() fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1).Return(index.FileInfo{1, 1}, nil) fx.index.EXPECT().FileInfo(ctx, storeKey, fileId2).Return(index.FileInfo{2, 2}, nil) diff --git a/index/index.go b/index/index.go index 28fa065..147376a 100644 --- a/index/index.go +++ b/index/index.go @@ -10,7 +10,9 @@ import ( ) var ( - ErrCidsNotExist = errors.New("cids not exist") + ErrCidsNotExist = errors.New("cids not exist") + ErrTargetStorageExists = errors.New("target storage exists") + ErrStorageNotFound = errors.New("storage not found") ) type Index interface { @@ -26,6 +28,7 @@ type Index interface { StorageSize(ctx context.Context, key string) (size uint64, err error) Lock(ctx context.Context, ks []cid.Cid) (unlock func(), err error) AddBlocks(ctx context.Context, upload []blocks.Block) error + MoveStorage(ctx context.Context, fromKey, toKey string) error app.Component } diff --git a/index/mock_index/mock_index.go b/index/mock_index/mock_index.go index dfdf9ae..bd82cbe 100644 --- a/index/mock_index/mock_index.go +++ b/index/mock_index/mock_index.go @@ -184,6 +184,20 @@ func (mr *MockIndexMockRecorder) Lock(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockIndex)(nil).Lock), arg0, arg1) } +// MoveStorage mocks base method. +func (m *MockIndex) MoveStorage(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MoveStorage", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// MoveStorage indicates an expected call of MoveStorage. +func (mr *MockIndexMockRecorder) MoveStorage(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockIndex)(nil).MoveStorage), arg0, arg1, arg2) +} + // Name mocks base method. func (m *MockIndex) Name() string { m.ctrl.T.Helper() diff --git a/index/redisindex/redisindex.go b/index/redisindex/redisindex.go index fe8d25a..735bde5 100644 --- a/index/redisindex/redisindex.go +++ b/index/redisindex/redisindex.go @@ -205,6 +205,26 @@ func (r *redisIndex) FileInfo(ctx context.Context, key, fileId string) (info ind return } +func (r *redisIndex) MoveStorage(ctx context.Context, fromKey, toKey string) (err error) { + ok, err := r.cl.RenameNX(ctx, storageKey(fromKey), storageKey(toKey)).Result() + if err != nil { + if err.Error() == "ERR no such key" { + return index.ErrStorageNotFound + } + return err + } + if !ok { + ex, err := r.cl.Exists(ctx, storageKey(toKey)).Result() + if err != nil { + return err + } + if ex > 0 { + return index.ErrTargetStorageExists + } + } + return +} + func (r *redisIndex) cidsAddRef(ctx context.Context, cids []CidInfo) error { now := time.Now() for _, c := range cids { @@ -339,8 +359,8 @@ func (r *redisIndex) cidInfoByByteKeys(ctx context.Context, ks [][]byte) (info [ return r.cidInfoByKeys(ctx, cids) } -func storageKey(spaceId string) string { - return "s:" + spaceId +func storageKey(storeKey string) string { + return "s:" + storeKey } func fileIdKey(fileId string) string { diff --git a/index/redisindex/redisindex_test.go b/index/redisindex/redisindex_test.go index a7268b0..902f53d 100644 --- a/index/redisindex/redisindex_test.go +++ b/index/redisindex/redisindex_test.go @@ -2,6 +2,7 @@ package redisindex import ( "context" + "github.com/anyproto/any-sync-filenode/index" "github.com/anyproto/any-sync-filenode/redisprovider/testredisprovider" "github.com/anyproto/any-sync-filenode/testutil" "github.com/anyproto/any-sync/app" @@ -127,6 +128,31 @@ func TestRedisIndex_Lock(t *testing.T) { unlock() } +func TestRedisIndex_MoveStorage(t *testing.T) { + const ( + oldKey = "oldKey" + newKey = "newKey" + ) + t.Run("success", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1))) + require.NoError(t, fx.MoveStorage(ctx, oldKey, newKey)) + }) + t.Run("err storage not found", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrStorageNotFound.Error()) + }) + t.Run("err taget exists", func(t *testing.T) { + fx := newFixture(t) + defer fx.Finish(t) + require.NoError(t, fx.Bind(ctx, oldKey, "fid", testutil.NewRandBlocks(1))) + require.NoError(t, fx.Bind(ctx, newKey, "fid", testutil.NewRandBlocks(1))) + assert.EqualError(t, fx.MoveStorage(ctx, oldKey, newKey), index.ErrTargetStorageExists.Error()) + }) +} + func Test100KCids(t *testing.T) { t.Skip() fx := newFixture(t)