diff --git a/piecestorage/filestore.go b/piecestorage/filestore.go index 543262c6..0d83afc8 100644 --- a/piecestorage/filestore.go +++ b/piecestorage/filestore.go @@ -6,6 +6,7 @@ import ( "io" "os" "path" + "path/filepath" "github.com/filecoin-project/dagstore/mount" @@ -22,28 +23,40 @@ type fsPieceStorage struct { } func (f *fsPieceStorage) Len(_ context.Context, resourceId string) (int64, error) { - st, err := os.Stat(path.Join(f.baseUrl, resourceId)) + size := int64(-1) + err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error { + if info.Name() == resourceId { + if info.IsDir() { + return fmt.Errorf("resource %s expect to be a file but found directory", resourceId) + } + size = info.Size() + + return filepath.SkipDir + } + return nil + }) if err != nil { return 0, err } - - if st.IsDir() { - return 0, fmt.Errorf("resource %s expect to be a file but found directory", resourceId) + if size == -1 { + return 0, fmt.Errorf("resource %s not found", resourceId) } - return st.Size(), err + + return size, nil } func (f *fsPieceStorage) ListResourceIds(_ context.Context) ([]string, error) { - entries, err := os.ReadDir(f.baseUrl) - if err != nil { - return nil, err - } var resources []string - for _, entry := range entries { - if !entry.IsDir() { - resources = append(resources, entry.Name()) + err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error { + if !info.IsDir() { + resources = append(resources, info.Name()) } + return nil + }) + if err != nil { + return nil, err } + return resources, nil } @@ -67,8 +80,33 @@ func (f *fsPieceStorage) SaveTo(_ context.Context, resourceId string, r io.Reade return wlen, err } +func (f *fsPieceStorage) findFile(resourceId string) (string, error) { + var dstPath string + err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error { + if info.Name() == resourceId { + if info.IsDir() { + return fmt.Errorf("resource %s expect to be a file but found directory", resourceId) + } + dstPath = path + return filepath.SkipDir + } + return nil + }) + if err != nil { + return "", err + } + if len(dstPath) == 0 { + return "", fmt.Errorf("resource %s not found", resourceId) + } + + return dstPath, nil +} + func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) (io.ReadCloser, error) { - dstPath := path.Join(f.baseUrl, resourceId) + dstPath, err := f.findFile(resourceId) + if err != nil { + return nil, err + } fs, err := os.Open(dstPath) if err != nil { return nil, fmt.Errorf("unable to open file %s %w", dstPath, err) @@ -77,7 +115,10 @@ func (f *fsPieceStorage) GetReaderCloser(_ context.Context, resourceId string) ( } func (f *fsPieceStorage) GetMountReader(_ context.Context, resourceId string) (mount.Reader, error) { - dstPath := path.Join(f.baseUrl, resourceId) + dstPath, err := f.findFile(resourceId) + if err != nil { + return nil, err + } fs, err := os.Open(dstPath) if err != nil { return nil, fmt.Errorf("unable to open file %s %w", dstPath, err) @@ -101,18 +142,24 @@ func (f *fsPieceStorage) GetPieceTransfer(_ context.Context, pieceCid string) (s } func (f *fsPieceStorage) Has(_ context.Context, resourceId string) (bool, error) { - s, err := os.Stat(path.Join(f.baseUrl, resourceId)) - if err != nil { - if os.IsNotExist(err) { - return false, nil + var has bool + err := filepath.Walk(f.baseUrl, func(path string, info os.FileInfo, _ error) error { + if info.Name() == resourceId { + if info.IsDir() { + return fmt.Errorf("resource %s expect to be a file but found directory", resourceId) + } + if info.Mode().IsRegular() { + has = true + } + return filepath.SkipDir } + return nil + }) + if err != nil { return false, err } - if !s.Mode().IsRegular() { - return false, nil - } - return true, nil + return has, nil } func (f *fsPieceStorage) Validate(_ string) error { diff --git a/piecestorage/filestore_test.go b/piecestorage/filestore_test.go index 447dec00..07160026 100644 --- a/piecestorage/filestore_test.go +++ b/piecestorage/filestore_test.go @@ -3,14 +3,17 @@ package piecestorage import ( "context" "crypto/rand" + "fmt" "io" "os" path2 "path" + "path/filepath" "testing" "github.com/google/uuid" "github.com/ipfs-force-community/droplet/v2/config" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -19,8 +22,8 @@ func TestReWrite(t *testing.T) { path := path2.Join(os.TempDir(), uuid.New().String()) _ = os.MkdirAll(path, os.ModePerm) name := "market-test-tmp" - filepath := path2.Join(path, name) - _ = os.Remove(filepath) + filePath := path2.Join(path, name) + _ = os.Remove(filePath) ctx := context.TODO() ifs, err := NewFsPieceStorage(&config.FsPieceStorage{ReadOnly: false, Path: path}) @@ -29,7 +32,7 @@ func TestReWrite(t *testing.T) { require.NoErrorf(t, err, "expect to write file ") require.Equal(t, wlen, int64(100)) - fs, err := os.Open(filepath) + fs, err := os.Open(filePath) if err != nil { if !os.IsExist(err) { require.NoErrorf(t, err, "expect file exit") @@ -42,36 +45,88 @@ func TestReWrite(t *testing.T) { require.Equal(t, int64(len(buf)), int64(100)) } - _, err = ifs.GetStorageStatus() - require.Nil(t, err) - require.Equal(t, FS, ifs.Type()) - length, err := ifs.Len(ctx, name) - require.NoError(t, err, "fail to get length") - require.Equal(t, int64(100), length) - - has, err := ifs.Has(ctx, name) - require.NoError(t, err, "fail to check file exit") - require.True(t, has) - require.False(t, ifs.ReadOnly()) - - resources, err := ifs.ListResourceIds(ctx) - require.NoErrorf(t, err, "expect resource but got err") - require.Len(t, resources, 1) - require.Equal(t, resources[0], name) - - readerCloser, err := ifs.GetReaderCloser(ctx, name) - require.NoError(t, err, "fail to get reader closer") - buf, err = io.ReadAll(readerCloser) - require.NoErrorf(t, err, "expect read file success") - if len(buf) != 100 { - require.Equal(t, int64(len(buf)), int64(100)) + files := []string{"f1", "f2", "f3", "f4"} + data, err := io.ReadAll(io.LimitReader(rand.Reader, 100)) + require.NoError(t, err) + for i, f := range files { + if i%2 == 0 { + _ = os.MkdirAll(filepath.Join(path, "tmp"), os.ModePerm) + assert.NoError(t, os.WriteFile(filepath.Join(path, "tmp", f), data, os.ModePerm)) + continue + } + wlen, err := ifs.SaveTo(ctx, f, io.LimitReader(rand.Reader, 100)) + assert.NoErrorf(t, err, "expect to write file ") + assert.Equal(t, int64(100), wlen) } - mounterReader, err := ifs.GetMountReader(ctx, name) - require.NoError(t, err, "fail to get mount reader") - buf, err = io.ReadAll(mounterReader) - require.NoErrorf(t, err, "expect read file success") - if len(buf) != 100 { - require.Equal(t, int64(len(buf)), int64(100)) + for _, name := range append(files, name) { + _, err = ifs.GetStorageStatus() + require.Nil(t, err) + require.Equal(t, FS, ifs.Type()) + + length, err := ifs.Len(ctx, name) + require.NoError(t, err, "fail to get length") + require.Equal(t, int64(100), length) + + has, err := ifs.Has(ctx, name) + require.NoError(t, err, "fail to check file exit") + require.True(t, has) + require.False(t, ifs.ReadOnly()) + + resources, err := ifs.ListResourceIds(ctx) + require.NoErrorf(t, err, "expect resource but got err") + require.Len(t, resources, 5) + require.Contains(t, resources, name) + + readerCloser, err := ifs.GetReaderCloser(ctx, name) + require.NoError(t, err, "fail to get reader closer") + buf, err = io.ReadAll(readerCloser) + require.NoErrorf(t, err, "expect read file success") + if len(buf) != 100 { + require.Equal(t, int64(len(buf)), int64(100)) + } + + mounterReader, err := ifs.GetMountReader(ctx, name) + require.NoError(t, err, "fail to get mount reader") + buf, err = io.ReadAll(mounterReader) + require.NoErrorf(t, err, "expect read file success") + if len(buf) != 100 { + require.Equal(t, int64(len(buf)), int64(100)) + } } + + dir := "tmp" + expectErr := fmt.Errorf("resource %s expect to be a file but found directory", dir) + has, err := ifs.Has(ctx, dir) + require.Equal(t, expectErr, err) + require.False(t, has) + + length, err := ifs.Len(ctx, dir) + require.Equal(t, expectErr, err) + assert.Equal(t, int64(0), length) + + readerCloser, err := ifs.GetReaderCloser(ctx, dir) + require.Equal(t, expectErr, err) + assert.Nil(t, readerCloser) + + mounterReader, err := ifs.GetMountReader(ctx, dir) + require.Equal(t, expectErr, err) + assert.Nil(t, mounterReader) + + noExistFile := "f111" + has, err = ifs.Has(ctx, noExistFile) + require.NoError(t, err) + require.False(t, has) + + length, err = ifs.Len(ctx, noExistFile) + require.Error(t, err) + assert.Equal(t, int64(0), length) + + readerCloser, err = ifs.GetReaderCloser(ctx, noExistFile) + require.Error(t, err) + assert.Nil(t, readerCloser) + + mounterReader, err = ifs.GetMountReader(ctx, noExistFile) + require.Error(t, err) + assert.Nil(t, mounterReader) }