Skip to content

Commit

Permalink
feat: piecestorage support recursive lookup file
Browse files Browse the repository at this point in the history
  • Loading branch information
@simlecode
simlecode committed Dec 12, 2023
1 parent 6a7d921 commit e70dfb0
Show file tree
Hide file tree
Showing 2 changed files with 156 additions and 54 deletions.
91 changes: 69 additions & 22 deletions piecestorage/filestore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io"
"os"
"path"
"path/filepath"

"github.com/filecoin-project/dagstore/mount"

Expand All @@ -22,28 +23,40 @@ type fsPieceStorage struct {
}

func (f *fsPieceStorage) Len(_ context.Context, resourceId string) (int64, error) {
st, err := os.Stat(path.Join(f.baseUrl, resourceId))
size := int64(-1)
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
size = info.Size()

return filepath.SkipDir
}
return nil
})
if err != nil {
return 0, err
}

if st.IsDir() {
return 0, fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
if size == -1 {
return 0, fmt.Errorf("resource %s not found", resourceId)
}
return st.Size(), err

return size, nil
}

func (f *fsPieceStorage) ListResourceIds(_ context.Context) ([]string, error) {
entries, err := os.ReadDir(f.baseUrl)
if err != nil {
return nil, err
}
var resources []string
for _, entry := range entries {
if !entry.IsDir() {
resources = append(resources, entry.Name())
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if !info.IsDir() {
resources = append(resources, info.Name())
}
return nil
})
if err != nil {
return nil, err
}

return resources, nil
}

Expand All @@ -67,8 +80,33 @@ func (f *fsPieceStorage) SaveTo(_ context.Context, resourceId string, r io.Reade
return wlen, err
}

func (f *fsPieceStorage) findFile(resourceId string) (string, error) {
var dstPath string
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
dstPath = path
return filepath.SkipDir
}
return nil
})
if err != nil {
return "", err
}
if len(dstPath) == 0 {
return "", fmt.Errorf("resource %s not found", resourceId)
}

return dstPath, nil
}

func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) (io.ReadCloser, error) {
dstPath := path.Join(f.baseUrl, resourceId)
dstPath, err := f.findFile(resourceId)
if err != nil {
return nil, err
}
fs, err := os.Open(dstPath)
if err != nil {
return nil, fmt.Errorf("unable to open file %s %w", dstPath, err)
Expand All @@ -77,7 +115,10 @@ func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) (
}

func (f *fsPieceStorage) GetMountReader(_ context.Context, resourceId string) (mount.Reader, error) {
dstPath := path.Join(f.baseUrl, resourceId)
dstPath, err := f.findFile(resourceId)
if err != nil {
return nil, err
}
fs, err := os.Open(dstPath)
if err != nil {
return nil, fmt.Errorf("unable to open file %s %w", dstPath, err)
Expand All @@ -101,18 +142,24 @@ func (f *fsPieceStorage) GetPieceTransfer(_ context.Context, pieceCid string) (s
}

func (f *fsPieceStorage) Has(_ context.Context, resourceId string) (bool, error) {
s, err := os.Stat(path.Join(f.baseUrl, resourceId))
if err != nil {
if os.IsNotExist(err) {
return false, nil
var has bool
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
if info.Mode().IsRegular() {
has = true
}
return filepath.SkipDir
}
return nil
})
if err != nil {
return false, err
}
if !s.Mode().IsRegular() {
return false, nil
}

return true, nil
return has, nil
}

func (f *fsPieceStorage) Validate(_ string) error {
Expand Down
119 changes: 87 additions & 32 deletions piecestorage/filestore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@ package piecestorage
import (
"context"
"crypto/rand"
"fmt"
"io"
"os"
path2 "path"
"path/filepath"
"testing"

"github.com/google/uuid"

"github.com/ipfs-force-community/droplet/v2/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

Expand All @@ -19,8 +22,8 @@ func TestReWrite(t *testing.T) {
path := path2.Join(os.TempDir(), uuid.New().String())
_ = os.MkdirAll(path, os.ModePerm)
name := "market-test-tmp"
filepath := path2.Join(path, name)
_ = os.Remove(filepath)
filePath := path2.Join(path, name)
_ = os.Remove(filePath)

ctx := context.TODO()
ifs, err := NewFsPieceStorage(&config.FsPieceStorage{ReadOnly: false, Path: path})
Expand All @@ -29,7 +32,7 @@ func TestReWrite(t *testing.T) {

require.NoErrorf(t, err, "expect to write file ")
require.Equal(t, wlen, int64(100))
fs, err := os.Open(filepath)
fs, err := os.Open(filePath)
if err != nil {
if !os.IsExist(err) {
require.NoErrorf(t, err, "expect file exit")
Expand All @@ -42,36 +45,88 @@ func TestReWrite(t *testing.T) {
require.Equal(t, int64(len(buf)), int64(100))
}

_, err = ifs.GetStorageStatus()
require.Nil(t, err)
require.Equal(t, FS, ifs.Type())
length, err := ifs.Len(ctx, name)
require.NoError(t, err, "fail to get length")
require.Equal(t, int64(100), length)

has, err := ifs.Has(ctx, name)
require.NoError(t, err, "fail to check file exit")
require.True(t, has)
require.False(t, ifs.ReadOnly())

resources, err := ifs.ListResourceIds(ctx)
require.NoErrorf(t, err, "expect resource but got err")
require.Len(t, resources, 1)
require.Equal(t, resources[0], name)

readerCloser, err := ifs.GetReaderCloser(ctx, name)
require.NoError(t, err, "fail to get reader closer")
buf, err = io.ReadAll(readerCloser)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
files := []string{"f1", "f2", "f3", "f4"}
data, err := io.ReadAll(io.LimitReader(rand.Reader, 100))
require.NoError(t, err)
for i, f := range files {
if i%2 == 0 {
_ = os.MkdirAll(filepath.Join(path, "tmp"), os.ModePerm)
assert.NoError(t, os.WriteFile(filepath.Join(path, "tmp", f), data, os.ModePerm))
continue
}
wlen, err := ifs.SaveTo(ctx, f, io.LimitReader(rand.Reader, 100))
assert.NoErrorf(t, err, "expect to write file ")
assert.Equal(t, int64(100), wlen)
}

mounterReader, err := ifs.GetMountReader(ctx, name)
require.NoError(t, err, "fail to get mount reader")
buf, err = io.ReadAll(mounterReader)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
for _, name := range append(files, name) {
_, err = ifs.GetStorageStatus()
require.Nil(t, err)
require.Equal(t, FS, ifs.Type())

length, err := ifs.Len(ctx, name)
require.NoError(t, err, "fail to get length")
require.Equal(t, int64(100), length)

has, err := ifs.Has(ctx, name)
require.NoError(t, err, "fail to check file exit")
require.True(t, has)
require.False(t, ifs.ReadOnly())

resources, err := ifs.ListResourceIds(ctx)
require.NoErrorf(t, err, "expect resource but got err")
require.Len(t, resources, 5)
require.Contains(t, resources, name)

readerCloser, err := ifs.GetReaderCloser(ctx, name)
require.NoError(t, err, "fail to get reader closer")
buf, err = io.ReadAll(readerCloser)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
}

mounterReader, err := ifs.GetMountReader(ctx, name)
require.NoError(t, err, "fail to get mount reader")
buf, err = io.ReadAll(mounterReader)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
}
}

dir := "tmp"
expectErr := fmt.Errorf("resource %s expect to be a file but found directory", dir)
has, err := ifs.Has(ctx, dir)
require.Equal(t, expectErr, err)
require.False(t, has)

length, err := ifs.Len(ctx, dir)
require.Equal(t, expectErr, err)
assert.Equal(t, int64(0), length)

readerCloser, err := ifs.GetReaderCloser(ctx, dir)
require.Equal(t, expectErr, err)
assert.Nil(t, readerCloser)

mounterReader, err := ifs.GetMountReader(ctx, dir)
require.Equal(t, expectErr, err)
assert.Nil(t, mounterReader)

noExistFile := "f111"
has, err = ifs.Has(ctx, noExistFile)
require.NoError(t, err)
require.False(t, has)

length, err = ifs.Len(ctx, noExistFile)
require.Error(t, err)
assert.Equal(t, int64(0), length)

readerCloser, err = ifs.GetReaderCloser(ctx, noExistFile)
require.Error(t, err)
assert.Nil(t, readerCloser)

mounterReader, err = ifs.GetMountReader(ctx, noExistFile)
require.Error(t, err)
assert.Nil(t, mounterReader)
}

0 comments on commit e70dfb0

Please sign in to comment.