Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: piecestorage support recursive lookup file #501

Merged
merged 1 commit into from
Dec 13, 2023
Merged

feat: piecestorage support recursive lookup file #501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 69 additions & 22 deletions piecestorage/filestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io"
"os"
"path"
"path/filepath"

"github.com/filecoin-project/dagstore/mount"

Expand All @@ -22,28 +23,40 @@ type fsPieceStorage struct {
}

func (f *fsPieceStorage) Len(_ context.Context, resourceId string) (int64, error) {
st, err := os.Stat(path.Join(f.baseUrl, resourceId))
size := int64(-1)
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
size = info.Size()

return filepath.SkipDir
}
return nil
})
if err != nil {
return 0, err
}

if st.IsDir() {
return 0, fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
if size == -1 {
return 0, fmt.Errorf("resource %s not found", resourceId)
}
return st.Size(), err

return size, nil
}

func (f *fsPieceStorage) ListResourceIds(_ context.Context) ([]string, error) {
entries, err := os.ReadDir(f.baseUrl)
if err != nil {
return nil, err
}
var resources []string
for _, entry := range entries {
if !entry.IsDir() {
resources = append(resources, entry.Name())
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if !info.IsDir() {
resources = append(resources, info.Name())
LinZexiao marked this conversation as resolved.
Show resolved Hide resolved
}
return nil
})
if err != nil {
return nil, err
}

return resources, nil
}

Expand All @@ -67,8 +80,33 @@ func (f *fsPieceStorage) SaveTo(_ context.Context, resourceId string, r io.Reade
return wlen, err
}

func (f *fsPieceStorage) findFile(resourceId string) (string, error) {
var dstPath string
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
dstPath = path
return filepath.SkipDir
}
return nil
})
if err != nil {
return "", err
}
if len(dstPath) == 0 {
return "", fmt.Errorf("resource %s not found", resourceId)
}

return dstPath, nil
}

func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) (io.ReadCloser, error) {
dstPath := path.Join(f.baseUrl, resourceId)
dstPath, err := f.findFile(resourceId)
if err != nil {
return nil, err
}
fs, err := os.Open(dstPath)
if err != nil {
return nil, fmt.Errorf("unable to open file %s %w", dstPath, err)
Expand All @@ -77,7 +115,10 @@ func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) (
}

func (f *fsPieceStorage) GetMountReader(_ context.Context, resourceId string) (mount.Reader, error) {
dstPath := path.Join(f.baseUrl, resourceId)
dstPath, err := f.findFile(resourceId)
if err != nil {
return nil, err
}
fs, err := os.Open(dstPath)
if err != nil {
return nil, fmt.Errorf("unable to open file %s %w", dstPath, err)
Expand All @@ -101,18 +142,24 @@ func (f *fsPieceStorage) GetPieceTransfer(_ context.Context, pieceCid string) (s
}

func (f *fsPieceStorage) Has(_ context.Context, resourceId string) (bool, error) {
s, err := os.Stat(path.Join(f.baseUrl, resourceId))
if err != nil {
if os.IsNotExist(err) {
return false, nil
var has bool
err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error {
if info.Name() == resourceId {
if info.IsDir() {
return fmt.Errorf("resource %s expect to be a file but found directory", resourceId)
}
if info.Mode().IsRegular() {
has = true
}
return filepath.SkipDir
}
return nil
})
LinZexiao marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return false, err
}
if !s.Mode().IsRegular() {
return false, nil
}

return true, nil
return has, nil
}

func (f *fsPieceStorage) Validate(_ string) error {
Expand Down
119 changes: 87 additions & 32 deletions piecestorage/filestore_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@ package piecestorage
import (
"context"
"crypto/rand"
"fmt"
"io"
"os"
path2 "path"
"path/filepath"
"testing"

"github.com/google/uuid"

"github.com/ipfs-force-community/droplet/v2/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

Expand All @@ -19,8 +22,8 @@ func TestReWrite(t *testing.T) {
path := path2.Join(os.TempDir(), uuid.New().String())
_ = os.MkdirAll(path, os.ModePerm)
name := "market-test-tmp"
filepath := path2.Join(path, name)
_ = os.Remove(filepath)
filePath := path2.Join(path, name)
_ = os.Remove(filePath)

ctx := context.TODO()
ifs, err := NewFsPieceStorage(&config.FsPieceStorage{ReadOnly: false, Path: path})
Expand All @@ -29,7 +32,7 @@ func TestReWrite(t *testing.T) {

require.NoErrorf(t, err, "expect to write file ")
require.Equal(t, wlen, int64(100))
fs, err := os.Open(filepath)
fs, err := os.Open(filePath)
if err != nil {
if !os.IsExist(err) {
require.NoErrorf(t, err, "expect file exit")
Expand All @@ -42,36 +45,88 @@ func TestReWrite(t *testing.T) {
require.Equal(t, int64(len(buf)), int64(100))
}

_, err = ifs.GetStorageStatus()
require.Nil(t, err)
require.Equal(t, FS, ifs.Type())
length, err := ifs.Len(ctx, name)
require.NoError(t, err, "fail to get length")
require.Equal(t, int64(100), length)

has, err := ifs.Has(ctx, name)
require.NoError(t, err, "fail to check file exit")
require.True(t, has)
require.False(t, ifs.ReadOnly())

resources, err := ifs.ListResourceIds(ctx)
require.NoErrorf(t, err, "expect resource but got err")
require.Len(t, resources, 1)
require.Equal(t, resources[0], name)

readerCloser, err := ifs.GetReaderCloser(ctx, name)
require.NoError(t, err, "fail to get reader closer")
buf, err = io.ReadAll(readerCloser)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
files := []string{"f1", "f2", "f3", "f4"}
data, err := io.ReadAll(io.LimitReader(rand.Reader, 100))
require.NoError(t, err)
for i, f := range files {
if i%2 == 0 {
_ = os.MkdirAll(filepath.Join(path, "tmp"), os.ModePerm)
assert.NoError(t, os.WriteFile(filepath.Join(path, "tmp", f), data, os.ModePerm))
continue
}
wlen, err := ifs.SaveTo(ctx, f, io.LimitReader(rand.Reader, 100))
assert.NoErrorf(t, err, "expect to write file ")
assert.Equal(t, int64(100), wlen)
}

mounterReader, err := ifs.GetMountReader(ctx, name)
require.NoError(t, err, "fail to get mount reader")
buf, err = io.ReadAll(mounterReader)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
for _, name := range append(files, name) {
_, err = ifs.GetStorageStatus()
require.Nil(t, err)
require.Equal(t, FS, ifs.Type())

length, err := ifs.Len(ctx, name)
require.NoError(t, err, "fail to get length")
require.Equal(t, int64(100), length)

has, err := ifs.Has(ctx, name)
require.NoError(t, err, "fail to check file exit")
require.True(t, has)
require.False(t, ifs.ReadOnly())

resources, err := ifs.ListResourceIds(ctx)
require.NoErrorf(t, err, "expect resource but got err")
require.Len(t, resources, 5)
require.Contains(t, resources, name)

readerCloser, err := ifs.GetReaderCloser(ctx, name)
require.NoError(t, err, "fail to get reader closer")
buf, err = io.ReadAll(readerCloser)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
}

mounterReader, err := ifs.GetMountReader(ctx, name)
require.NoError(t, err, "fail to get mount reader")
buf, err = io.ReadAll(mounterReader)
require.NoErrorf(t, err, "expect read file success")
if len(buf) != 100 {
require.Equal(t, int64(len(buf)), int64(100))
}
}

dir := "tmp"
expectErr := fmt.Errorf("resource %s expect to be a file but found directory", dir)
has, err := ifs.Has(ctx, dir)
require.Equal(t, expectErr, err)
require.False(t, has)

length, err := ifs.Len(ctx, dir)
require.Equal(t, expectErr, err)
assert.Equal(t, int64(0), length)

readerCloser, err := ifs.GetReaderCloser(ctx, dir)
require.Equal(t, expectErr, err)
assert.Nil(t, readerCloser)

mounterReader, err := ifs.GetMountReader(ctx, dir)
require.Equal(t, expectErr, err)
assert.Nil(t, mounterReader)

noExistFile := "f111"
has, err = ifs.Has(ctx, noExistFile)
require.NoError(t, err)
require.False(t, has)

length, err = ifs.Len(ctx, noExistFile)
require.Error(t, err)
assert.Equal(t, int64(0), length)

readerCloser, err = ifs.GetReaderCloser(ctx, noExistFile)
require.Error(t, err)
assert.Nil(t, readerCloser)

mounterReader, err = ifs.GetMountReader(ctx, noExistFile)
require.Error(t, err)
assert.Nil(t, mounterReader)
}
Loading