GO-2042 file limits by identity

.github/workflows/coverage.yml

@@ -35,6 +35,23 @@ jobs:
       - name: git config
         run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/
 
+      # check license {{
+      - name: Setup license repository
+        uses: actions/checkout@master
+        with:
+          repository: anyproto/open
+          ref: refs/heads/main
+          path: ./open
+      - name: Check licenses
+        run: |
+          cd open
+          python3 tools/generate.py --platform golang
+          cd ..
+          sudo gem install license_finder
+          license_finder inherited_decisions add open/decisions.yml
+          license_finder --enabled-package-managers gomodules
+      # }}
+
       - name: deps
         run: make deps CGO_ENABLED=0
 
@@ -46,6 +63,16 @@ jobs:
           TESTCOVERAGE_THRESHOLD: 0
         run: |
           go test ./... -coverprofile coverage.out -covermode count
+
+          generated_pattern='^\/\/ Code generated .* DO NOT EDIT\.$'
+          files_list=$(grep -rl "$generated_pattern" . | grep '\.go$' | sed 's/^\.\///')
+
+          for file in $files_list; do
+            echo "Removing $file from coverage report"
+            grep -v "$file" coverage.out > temp_file
+            mv temp_file coverage.out
+          done
+
           go tool cover -func coverage.out
           echo "Quality Gate: checking test coverage is above threshold ..."
           echo "Threshold             : $TESTCOVERAGE_THRESHOLD %"
@@ -58,3 +85,4 @@ jobs:
               echo "Failed"
               exit 1
           fi
+      - uses: seriousben/go-patch-cover-action@v1

filenode/filenode.go

@@ -68,31 +68,10 @@ func (fn *fileNode) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 }
 
 func (fn *fileNode) Add(ctx context.Context, spaceId string, fileId string, bs []blocks.Block) error {
-	// temporary code for migration
-	if fileId == "migration" {
-		return fn.migrate(ctx, bs)
-	}
-	if err := fn.ValidateSpaceId(ctx, spaceId, true); err != nil {
-		return err
-	}
-	unlock, err := fn.index.Lock(ctx, testutil.BlocksToKeys(bs))
-	if err != nil {
-		return err
-	}
-	defer unlock()
-	toUpload, err := fn.index.GetNonExistentBlocks(ctx, bs)
+	storeKey, err := fn.StoreKey(ctx, spaceId, true)
 	if err != nil {
 		return err
 	}
-	if len(toUpload) > 0 {
-		if err = fn.store.Add(ctx, toUpload); err != nil {
-			return err
-		}
-	}
-	return fn.index.Bind(ctx, spaceId, fileId, bs)
-}
-
-func (fn *fileNode) migrate(ctx context.Context, bs []blocks.Block) error {
 	unlock, err := fn.index.Lock(ctx, testutil.BlocksToKeys(bs))
 	if err != nil {
 		return err
@@ -107,19 +86,20 @@ func (fn *fileNode) migrate(ctx context.Context, bs []blocks.Block) error {
 			return err
 		}
 	}
-	return fn.index.AddBlocks(ctx, toUpload)
+	return fn.index.Bind(ctx, storeKey, fileId, bs)
 }
 
 func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid) (result []*fileproto.BlockAvailability, err error) {
+	var storeKey string
 	if spaceId != "" {
-		if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
+		if storeKey, err = fn.StoreKey(ctx, spaceId, false); err != nil {
 			return
 		}
 	}
 	result = make([]*fileproto.BlockAvailability, 0, len(cids))
 	var inSpaceM = make(map[string]struct{})
 	if spaceId != "" {
-		inSpace, err := fn.index.ExistsInSpace(ctx, spaceId, cids)
+		inSpace, err := fn.index.ExistsInStorage(ctx, storeKey, cids)
 		if err != nil {
 			return nil, err
 		}
@@ -148,33 +128,43 @@ func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid)
 }
 
 func (fn *fileNode) BlocksBind(ctx context.Context, spaceId, fileId string, cids ...cid.Cid) (err error) {
-	if err = fn.ValidateSpaceId(ctx, spaceId, true); err != nil {
+	storeKey, err := fn.StoreKey(ctx, spaceId, true)
+	if err != nil {
 		return err
 	}
 	unlock, err := fn.index.Lock(ctx, cids)
 	if err != nil {
 		return err
 	}
 	defer unlock()
-	return fn.index.BindCids(ctx, spaceId, fileId, cids)
+	return fn.index.BindCids(ctx, storeKey, fileId, cids)
 }
 
-func (fn *fileNode) ValidateSpaceId(ctx context.Context, spaceId string, checkLimit bool) (err error) {
+func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit bool) (storageKey string, err error) {
 	if spaceId == "" {
-		return fileprotoerr.ErrForbidden
+		return "", fileprotoerr.ErrForbidden
 	}
 	// this call also confirms that space exists and valid
-	limitBytes, err := fn.limit.Check(ctx, spaceId)
+	limitBytes, storageKey, err := fn.limit.Check(ctx, spaceId)
 	if err != nil {
 		return
 	}
+
+	if storageKey != spaceId {
+		// try to move store to the new key
+		mErr := fn.index.MoveStorage(ctx, spaceId, storageKey)
+		if mErr != nil && mErr != index.ErrStorageNotFound && mErr != index.ErrTargetStorageExists {
+			return "", mErr
+		}
+	}
+
 	if checkLimit {
-		currentSize, e := fn.index.SpaceSize(ctx, spaceId)
+		currentSize, e := fn.index.StorageSize(ctx, storageKey)
 		if e != nil {
-			return e
+			return "", e
 		}
 		if currentSize >= limitBytes {
-			return fileprotoerr.ErrSpaceLimitExceeded
+			return "", fileprotoerr.ErrSpaceLimitExceeded
 		}
 	}
 	return
@@ -183,13 +173,14 @@ func (fn *fileNode) ValidateSpaceId(ctx context.Context, spaceId string, checkLi
 func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *fileproto.SpaceInfoResponse, err error) {
 	info = &fileproto.SpaceInfoResponse{}
 	// we have space/identity validation in limit.Check
-	if info.LimitBytes, err = fn.limit.Check(ctx, spaceId); err != nil {
+	var storageKey string
+	if info.LimitBytes, storageKey, err = fn.limit.Check(ctx, spaceId); err != nil {
 		return nil, err
 	}
-	if info.UsageBytes, err = fn.index.SpaceSize(ctx, spaceId); err != nil {
+	if info.UsageBytes, err = fn.index.StorageSize(ctx, storageKey); err != nil {
 		return nil, err
 	}
-	si, err := fn.index.SpaceInfo(ctx, spaceId)
+	si, err := fn.index.StorageInfo(ctx, storageKey)
 	if err != nil {
 		return nil, err
 	}
@@ -199,22 +190,24 @@ func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *filepr
 }
 
 func (fn *fileNode) FilesDelete(ctx context.Context, spaceId string, fileIds []string) (err error) {
-	if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
+	storeKey, err := fn.StoreKey(ctx, spaceId, false)
+	if err != nil {
 		return
 	}
 	for _, fileId := range fileIds {
-		if err = fn.index.UnBind(ctx, spaceId, fileId); err != nil {
+		if err = fn.index.UnBind(ctx, storeKey, fileId); err != nil {
 			return
 		}
 	}
 	return
 }
 
 func (fn *fileNode) FileInfo(ctx context.Context, spaceId, fileId string) (info *fileproto.FileInfo, err error) {
-	if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
+	storeKey, err := fn.StoreKey(ctx, spaceId, false)
+	if err != nil {
 		return
 	}
-	fi, err := fn.index.FileInfo(ctx, spaceId, fileId)
+	fi, err := fn.index.FileInfo(ctx, storeKey, fileId)
 	if err != nil {
 		return nil, err
 	}

filenode/filenode_test.go

@@ -31,17 +31,19 @@ func TestFileNode_Add(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
 		var (
-			spaceId = testutil.NewRandSpaceId()
-			fileId  = testutil.NewRandCid().String()
-			b       = testutil.NewRandBlock(1024)
+			spaceId  = testutil.NewRandSpaceId()
+			storeKey = "sk:" + spaceId
+			fileId   = testutil.NewRandCid().String()
+			b        = testutil.NewRandBlock(1024)
 		)
 
-		fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
-		fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(120), nil)
+		fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
+		fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
+		fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(120), nil)
 		fx.index.EXPECT().Lock(ctx, []cid.Cid{b.Cid()}).Return(func() {}, nil)
 		fx.index.EXPECT().GetNonExistentBlocks(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil)
 		fx.store.EXPECT().Add(ctx, []blocks.Block{b})
-		fx.index.EXPECT().Bind(ctx, spaceId, fileId, []blocks.Block{b})
+		fx.index.EXPECT().Bind(ctx, storeKey, fileId, []blocks.Block{b})
 
 		resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{
 			SpaceId: spaceId,
@@ -56,13 +58,15 @@ func TestFileNode_Add(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
 		var (
-			spaceId = testutil.NewRandSpaceId()
-			fileId  = testutil.NewRandCid().String()
-			b       = testutil.NewRandBlock(1024)
+			spaceId  = testutil.NewRandSpaceId()
+			storeKey = "sk:" + spaceId
+			fileId   = testutil.NewRandCid().String()
+			b        = testutil.NewRandBlock(1024)
 		)
 
-		fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
-		fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(124), nil)
+		fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
+		fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
+		fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(124), nil)
 
 		resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{
 			SpaceId: spaceId,
@@ -146,13 +150,15 @@ func TestFileNode_Check(t *testing.T) {
 	fx := newFixture(t)
 	defer fx.Finish(t)
 	var spaceId = testutil.NewRandSpaceId()
+	var storeKey = "sk:" + spaceId
 	var bs = testutil.NewRandBlocks(3)
 	cids := make([][]byte, len(bs))
 	for _, b := range bs {
 		cids = append(cids, b.Cid().Bytes())
 	}
-	fx.limit.EXPECT().Check(ctx, spaceId)
-	fx.index.EXPECT().ExistsInSpace(ctx, spaceId, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil)
+	fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(100000), storeKey, nil)
+	fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
+	fx.index.EXPECT().ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil)
 	fx.index.EXPECT().Exists(ctx, bs[1].Cid()).Return(true, nil)
 	fx.index.EXPECT().Exists(ctx, bs[2].Cid()).Return(false, nil)
 	resp, err := fx.handler.BlocksCheck(ctx, &fileproto.BlocksCheckRequest{
@@ -170,21 +176,23 @@ func TestFileNode_BlocksBind(t *testing.T) {
 	fx := newFixture(t)
 	defer fx.Finish(t)
 	var (
-		spaceId = testutil.NewRandSpaceId()
-		fileId  = testutil.NewRandCid().String()
-		bs      = testutil.NewRandBlocks(3)
-		cidsB   = make([][]byte, len(bs))
-		cids    = make([]cid.Cid, len(bs))
+		spaceId  = testutil.NewRandSpaceId()
+		storeKey = "sk:" + spaceId
+		fileId   = testutil.NewRandCid().String()
+		bs       = testutil.NewRandBlocks(3)
+		cidsB    = make([][]byte, len(bs))
+		cids     = make([]cid.Cid, len(bs))
 	)
 	for i, b := range bs {
 		cids[i] = b.Cid()
 		cidsB[i] = b.Cid().Bytes()
 	}
 
-	fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
-	fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(12), nil)
+	fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
+	fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
+	fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(12), nil)
 	fx.index.EXPECT().Lock(ctx, cids).Return(func() {}, nil)
-	fx.index.EXPECT().BindCids(ctx, spaceId, fileId, cids)
+	fx.index.EXPECT().BindCids(ctx, storeKey, fileId, cids)
 
 	resp, err := fx.handler.BlocksBind(ctx, &fileproto.BlocksBindRequest{
 		SpaceId: spaceId,
@@ -200,13 +208,15 @@ func TestFileNode_FileInfo(t *testing.T) {
 	defer fx.Finish(t)
 
 	var (
-		spaceId = testutil.NewRandSpaceId()
-		fileId1 = testutil.NewRandCid().String()
-		fileId2 = testutil.NewRandCid().String()
+		spaceId  = testutil.NewRandSpaceId()
+		storeKey = "sk:" + spaceId
+		fileId1  = testutil.NewRandCid().String()
+		fileId2  = testutil.NewRandCid().String()
 	)
-	fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes()
-	fx.index.EXPECT().FileInfo(ctx, spaceId, fileId1).Return(index.FileInfo{1, 1}, nil)
-	fx.index.EXPECT().FileInfo(ctx, spaceId, fileId2).Return(index.FileInfo{2, 2}, nil)
+	fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes().Return(uint64(100000), storeKey, nil)
+	fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
+	fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1).Return(index.FileInfo{1, 1}, nil)
+	fx.index.EXPECT().FileInfo(ctx, storeKey, fileId2).Return(index.FileInfo{2, 2}, nil)
 
 	resp, err := fx.handler.FilesInfo(ctx, &fileproto.FilesInfoRequest{
 		SpaceId: spaceId,

filenode/rpchandler.go

@@ -166,7 +166,8 @@ func (r rpcHandler) FilesInfo(ctx context.Context, req *fileproto.FilesInfoReque
 	resp = &fileproto.FilesInfoResponse{
 		FilesInfo: make([]*fileproto.FileInfo, len(req.FileIds)),
 	}
-	if err = r.f.ValidateSpaceId(ctx, req.SpaceId, false); err != nil {
+	_, err = r.f.StoreKey(ctx, req.SpaceId, false)
+	if err != nil {
 		return nil, err
 	}
 	var info *fileproto.FileInfo

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/ahmetb/govvv v0.3.0
-	github.com/anyproto/any-sync v0.3.0
+	github.com/anyproto/any-sync v0.3.1
 	github.com/aws/aws-sdk-go v1.44.327
 	github.com/go-redsync/redsync/v4 v4.8.1
 	github.com/gogo/protobuf v1.3.2
@@ -14,8 +14,8 @@ require (
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/atomic v1.11.0
-	go.uber.org/mock v0.2.0
-	go.uber.org/zap v1.25.0
+	go.uber.org/mock v0.3.0
+	go.uber.org/zap v1.26.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -4,8 +4,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0=
 github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s=
-github.com/anyproto/any-sync v0.3.0 h1:4lVxHf6aA2BrefGqjzbyn/wnSuNoriU2o8IsABBMfJE=
-github.com/anyproto/any-sync v0.3.0/go.mod h1:8uddvzx/4Le7c9v5tb0jgma4GuAWfUDMTeRCBLu0MTo=
+github.com/anyproto/any-sync v0.3.1 h1:AHsIYyhM9J+eqKVjnsuGgT/4u+f47JyEhfIBMnDLIIA=
+github.com/anyproto/any-sync v0.3.1/go.mod h1:v0w3l3FBWjzNgg5t8aWlI+aYkcA8kLaoJFfr/GHsWYk=
 github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY=
 github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY=
 github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i6IA=
@@ -15,8 +15,6 @@ github.com/anyproto/go-slip21 v1.0.0/go.mod h1:gbIJt7HAdr5DuT4f2pFTKCBSUWYsm/fys
 github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY=
 github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
-github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
@@ -226,14 +224,14 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
-go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
-go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
+go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
+go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
-go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
-go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=