Skip to content

Commit

Permalink
Merge pull request #38 from anyproto/GO-2042-file-limits-by-identity
Browse files Browse the repository at this point in the history 
GO-2042 file limits by identity
  • Loading branch information
@cheggaaa
cheggaaa committed Sep 21, 2023
2 parents cbf3ffc + 83ced34 commit 2e00d26
Show file tree
Hide file tree
Showing 17 changed files with 303 additions and 178 deletions.
28 changes: 28 additions & 0 deletions .github/workflows/coverage.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,23 @@ jobs:
- name: git config
run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/

# check license {{
- name: Setup license repository
uses: actions/checkout@master
with:
repository: anyproto/open
ref: refs/heads/main
path: ./open
- name: Check licenses
run: |
cd open
python3 tools/generate.py --platform golang
cd ..
sudo gem install license_finder
license_finder inherited_decisions add open/decisions.yml
license_finder --enabled-package-managers gomodules
# }}

- name: deps
run: make deps CGO_ENABLED=0

Expand All @@ -46,6 +63,16 @@ jobs:
TESTCOVERAGE_THRESHOLD: 0
run: |
go test ./... -coverprofile coverage.out -covermode count
generated_pattern='^\/\/ Code generated .* DO NOT EDIT\.$'
files_list=$(grep -rl "$generated_pattern" . | grep '\.go$' | sed 's/^\.\///')
for file in $files_list; do
echo "Removing $file from coverage report"
grep -v "$file" coverage.out > temp_file
mv temp_file coverage.out
done
go tool cover -func coverage.out
echo "Quality Gate: checking test coverage is above threshold ..."
echo "Threshold : $TESTCOVERAGE_THRESHOLD %"
Expand All @@ -58,3 +85,4 @@ jobs:
echo "Failed"
exit 1
fi
- uses: seriousben/go-patch-cover-action@v1
73 changes: 33 additions & 40 deletions filenode/filenode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,31 +68,10 @@ func (fn *fileNode) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
}

func (fn *fileNode) Add(ctx context.Context, spaceId string, fileId string, bs []blocks.Block) error {
// temporary code for migration
if fileId == "migration" {
return fn.migrate(ctx, bs)
}
if err := fn.ValidateSpaceId(ctx, spaceId, true); err != nil {
return err
}
unlock, err := fn.index.Lock(ctx, testutil.BlocksToKeys(bs))
if err != nil {
return err
}
defer unlock()
toUpload, err := fn.index.GetNonExistentBlocks(ctx, bs)
storeKey, err := fn.StoreKey(ctx, spaceId, true)
if err != nil {
return err
}
if len(toUpload) > 0 {
if err = fn.store.Add(ctx, toUpload); err != nil {
return err
}
}
return fn.index.Bind(ctx, spaceId, fileId, bs)
}

func (fn *fileNode) migrate(ctx context.Context, bs []blocks.Block) error {
unlock, err := fn.index.Lock(ctx, testutil.BlocksToKeys(bs))
if err != nil {
return err
Expand All @@ -107,19 +86,20 @@ func (fn *fileNode) migrate(ctx context.Context, bs []blocks.Block) error {
return err
}
}
return fn.index.AddBlocks(ctx, toUpload)
return fn.index.Bind(ctx, storeKey, fileId, bs)
}

func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid) (result []*fileproto.BlockAvailability, err error) {
var storeKey string
if spaceId != "" {
if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
if storeKey, err = fn.StoreKey(ctx, spaceId, false); err != nil {
return
}
}
result = make([]*fileproto.BlockAvailability, 0, len(cids))
var inSpaceM = make(map[string]struct{})
if spaceId != "" {
inSpace, err := fn.index.ExistsInSpace(ctx, spaceId, cids)
inSpace, err := fn.index.ExistsInStorage(ctx, storeKey, cids)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -148,33 +128,43 @@ func (fn *fileNode) Check(ctx context.Context, spaceId string, cids ...cid.Cid)
}

func (fn *fileNode) BlocksBind(ctx context.Context, spaceId, fileId string, cids ...cid.Cid) (err error) {
if err = fn.ValidateSpaceId(ctx, spaceId, true); err != nil {
storeKey, err := fn.StoreKey(ctx, spaceId, true)
if err != nil {
return err
}
unlock, err := fn.index.Lock(ctx, cids)
if err != nil {
return err
}
defer unlock()
return fn.index.BindCids(ctx, spaceId, fileId, cids)
return fn.index.BindCids(ctx, storeKey, fileId, cids)
}

func (fn *fileNode) ValidateSpaceId(ctx context.Context, spaceId string, checkLimit bool) (err error) {
func (fn *fileNode) StoreKey(ctx context.Context, spaceId string, checkLimit bool) (storageKey string, err error) {
if spaceId == "" {
return fileprotoerr.ErrForbidden
return "", fileprotoerr.ErrForbidden
}
// this call also confirms that space exists and valid
limitBytes, err := fn.limit.Check(ctx, spaceId)
limitBytes, storageKey, err := fn.limit.Check(ctx, spaceId)
if err != nil {
return
}

if storageKey != spaceId {
// try to move store to the new key
mErr := fn.index.MoveStorage(ctx, spaceId, storageKey)
if mErr != nil && mErr != index.ErrStorageNotFound && mErr != index.ErrTargetStorageExists {
return "", mErr
}
}

if checkLimit {
currentSize, e := fn.index.SpaceSize(ctx, spaceId)
currentSize, e := fn.index.StorageSize(ctx, storageKey)
if e != nil {
return e
return "", e
}
if currentSize >= limitBytes {
return fileprotoerr.ErrSpaceLimitExceeded
return "", fileprotoerr.ErrSpaceLimitExceeded
}
}
return
Expand All @@ -183,13 +173,14 @@ func (fn *fileNode) ValidateSpaceId(ctx context.Context, spaceId string, checkLi
func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *fileproto.SpaceInfoResponse, err error) {
info = &fileproto.SpaceInfoResponse{}
// we have space/identity validation in limit.Check
if info.LimitBytes, err = fn.limit.Check(ctx, spaceId); err != nil {
var storageKey string
if info.LimitBytes, storageKey, err = fn.limit.Check(ctx, spaceId); err != nil {
return nil, err
}
if info.UsageBytes, err = fn.index.SpaceSize(ctx, spaceId); err != nil {
if info.UsageBytes, err = fn.index.StorageSize(ctx, storageKey); err != nil {
return nil, err
}
si, err := fn.index.SpaceInfo(ctx, spaceId)
si, err := fn.index.StorageInfo(ctx, storageKey)
if err != nil {
return nil, err
}
Expand All @@ -199,22 +190,24 @@ func (fn *fileNode) SpaceInfo(ctx context.Context, spaceId string) (info *filepr
}

func (fn *fileNode) FilesDelete(ctx context.Context, spaceId string, fileIds []string) (err error) {
if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
storeKey, err := fn.StoreKey(ctx, spaceId, false)
if err != nil {
return
}
for _, fileId := range fileIds {
if err = fn.index.UnBind(ctx, spaceId, fileId); err != nil {
if err = fn.index.UnBind(ctx, storeKey, fileId); err != nil {
return
}
}
return
}

func (fn *fileNode) FileInfo(ctx context.Context, spaceId, fileId string) (info *fileproto.FileInfo, err error) {
if err = fn.ValidateSpaceId(ctx, spaceId, false); err != nil {
storeKey, err := fn.StoreKey(ctx, spaceId, false)
if err != nil {
return
}
fi, err := fn.index.FileInfo(ctx, spaceId, fileId)
fi, err := fn.index.FileInfo(ctx, storeKey, fileId)
if err != nil {
return nil, err
}
Expand Down
64 changes: 37 additions & 27 deletions filenode/filenode_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,19 @@ func TestFileNode_Add(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
var (
spaceId = testutil.NewRandSpaceId()
fileId = testutil.NewRandCid().String()
b = testutil.NewRandBlock(1024)
spaceId = testutil.NewRandSpaceId()
storeKey = "sk:" + spaceId
fileId = testutil.NewRandCid().String()
b = testutil.NewRandBlock(1024)
)

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(120), nil)
fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(120), nil)
fx.index.EXPECT().Lock(ctx, []cid.Cid{b.Cid()}).Return(func() {}, nil)
fx.index.EXPECT().GetNonExistentBlocks(ctx, []blocks.Block{b}).Return([]blocks.Block{b}, nil)
fx.store.EXPECT().Add(ctx, []blocks.Block{b})
fx.index.EXPECT().Bind(ctx, spaceId, fileId, []blocks.Block{b})
fx.index.EXPECT().Bind(ctx, storeKey, fileId, []blocks.Block{b})

resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{
SpaceId: spaceId,
Expand All @@ -56,13 +58,15 @@ func TestFileNode_Add(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
var (
spaceId = testutil.NewRandSpaceId()
fileId = testutil.NewRandCid().String()
b = testutil.NewRandBlock(1024)
spaceId = testutil.NewRandSpaceId()
storeKey = "sk:" + spaceId
fileId = testutil.NewRandCid().String()
b = testutil.NewRandBlock(1024)
)

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(124), nil)
fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(124), nil)

resp, err := fx.handler.BlockPush(ctx, &fileproto.BlockPushRequest{
SpaceId: spaceId,
Expand Down Expand Up @@ -146,13 +150,15 @@ func TestFileNode_Check(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
var spaceId = testutil.NewRandSpaceId()
var storeKey = "sk:" + spaceId
var bs = testutil.NewRandBlocks(3)
cids := make([][]byte, len(bs))
for _, b := range bs {
cids = append(cids, b.Cid().Bytes())
}
fx.limit.EXPECT().Check(ctx, spaceId)
fx.index.EXPECT().ExistsInSpace(ctx, spaceId, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil)
fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(100000), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().ExistsInStorage(ctx, storeKey, testutil.BlocksToKeys(bs)).Return(testutil.BlocksToKeys(bs[:1]), nil)
fx.index.EXPECT().Exists(ctx, bs[1].Cid()).Return(true, nil)
fx.index.EXPECT().Exists(ctx, bs[2].Cid()).Return(false, nil)
resp, err := fx.handler.BlocksCheck(ctx, &fileproto.BlocksCheckRequest{
Expand All @@ -170,21 +176,23 @@ func TestFileNode_BlocksBind(t *testing.T) {
fx := newFixture(t)
defer fx.Finish(t)
var (
spaceId = testutil.NewRandSpaceId()
fileId = testutil.NewRandCid().String()
bs = testutil.NewRandBlocks(3)
cidsB = make([][]byte, len(bs))
cids = make([]cid.Cid, len(bs))
spaceId = testutil.NewRandSpaceId()
storeKey = "sk:" + spaceId
fileId = testutil.NewRandCid().String()
bs = testutil.NewRandBlocks(3)
cidsB = make([][]byte, len(bs))
cids = make([]cid.Cid, len(bs))
)
for i, b := range bs {
cids[i] = b.Cid()
cidsB[i] = b.Cid().Bytes()
}

fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), nil)
fx.index.EXPECT().SpaceSize(ctx, spaceId).Return(uint64(12), nil)
fx.limit.EXPECT().Check(ctx, spaceId).Return(uint64(123), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().StorageSize(ctx, storeKey).Return(uint64(12), nil)
fx.index.EXPECT().Lock(ctx, cids).Return(func() {}, nil)
fx.index.EXPECT().BindCids(ctx, spaceId, fileId, cids)
fx.index.EXPECT().BindCids(ctx, storeKey, fileId, cids)

resp, err := fx.handler.BlocksBind(ctx, &fileproto.BlocksBindRequest{
SpaceId: spaceId,
Expand All @@ -200,13 +208,15 @@ func TestFileNode_FileInfo(t *testing.T) {
defer fx.Finish(t)

var (
spaceId = testutil.NewRandSpaceId()
fileId1 = testutil.NewRandCid().String()
fileId2 = testutil.NewRandCid().String()
spaceId = testutil.NewRandSpaceId()
storeKey = "sk:" + spaceId
fileId1 = testutil.NewRandCid().String()
fileId2 = testutil.NewRandCid().String()
)
fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes()
fx.index.EXPECT().FileInfo(ctx, spaceId, fileId1).Return(index.FileInfo{1, 1}, nil)
fx.index.EXPECT().FileInfo(ctx, spaceId, fileId2).Return(index.FileInfo{2, 2}, nil)
fx.limit.EXPECT().Check(ctx, spaceId).AnyTimes().Return(uint64(100000), storeKey, nil)
fx.index.EXPECT().MoveStorage(ctx, spaceId, storeKey).AnyTimes()
fx.index.EXPECT().FileInfo(ctx, storeKey, fileId1).Return(index.FileInfo{1, 1}, nil)
fx.index.EXPECT().FileInfo(ctx, storeKey, fileId2).Return(index.FileInfo{2, 2}, nil)

resp, err := fx.handler.FilesInfo(ctx, &fileproto.FilesInfoRequest{
SpaceId: spaceId,
Expand Down
3 changes: 2 additions & 1 deletion filenode/rpchandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ func (r rpcHandler) FilesInfo(ctx context.Context, req *fileproto.FilesInfoReque
resp = &fileproto.FilesInfoResponse{
FilesInfo: make([]*fileproto.FileInfo, len(req.FileIds)),
}
if err = r.f.ValidateSpaceId(ctx, req.SpaceId, false); err != nil {
_, err = r.f.StoreKey(ctx, req.SpaceId, false)
if err != nil {
return nil, err
}
var info *fileproto.FileInfo
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.21

require (
github.com/ahmetb/govvv v0.3.0
github.com/anyproto/any-sync v0.3.0
github.com/anyproto/any-sync v0.3.1
github.com/aws/aws-sdk-go v1.44.327
github.com/go-redsync/redsync/v4 v4.8.1
github.com/gogo/protobuf v1.3.2
Expand All @@ -14,8 +14,8 @@ require (
github.com/redis/go-redis/v9 v9.1.0
github.com/stretchr/testify v1.8.4
go.uber.org/atomic v1.11.0
go.uber.org/mock v0.2.0
go.uber.org/zap v1.25.0
go.uber.org/mock v0.3.0
go.uber.org/zap v1.26.0
gopkg.in/yaml.v3 v3.0.1
)

Expand Down
14 changes: 6 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/ahmetb/govvv v0.3.0 h1:YGLGwEyiUwHFy5eh/RUhdupbuaCGBYn5T5GWXp+WJB0=
github.com/ahmetb/govvv v0.3.0/go.mod h1:4WRFpdWtc/YtKgPFwa1dr5+9hiRY5uKAL08bOlxOR6s=
github.com/anyproto/any-sync v0.3.0 h1:4lVxHf6aA2BrefGqjzbyn/wnSuNoriU2o8IsABBMfJE=
github.com/anyproto/any-sync v0.3.0/go.mod h1:8uddvzx/4Le7c9v5tb0jgma4GuAWfUDMTeRCBLu0MTo=
github.com/anyproto/any-sync v0.3.1 h1:AHsIYyhM9J+eqKVjnsuGgT/4u+f47JyEhfIBMnDLIIA=
github.com/anyproto/any-sync v0.3.1/go.mod h1:v0w3l3FBWjzNgg5t8aWlI+aYkcA8kLaoJFfr/GHsWYk=
github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY=
github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY=
github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i6IA=
Expand All @@ -15,8 +15,6 @@ github.com/anyproto/go-slip21 v1.0.0/go.mod h1:gbIJt7HAdr5DuT4f2pFTKCBSUWYsm/fys
github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY=
github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
Expand Down Expand Up @@ -226,14 +224,14 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down
Loading

0 comments on commit 2e00d26

Please sign in to comment.