diff --git a/.travis.yml b/.travis.yml index a3c59064..e944f594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ arch: go: - "1.14" - "1.15" + - "1.16" - tip os: diff --git a/README.md b/README.md index 1400bc4e..fb8eaaf8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ filesystem for full interoperability. * Support for compositional (union) file systems by combining multiple file systems acting as one * Specialized backends which modify existing filesystems (Read Only, Regexp filtered) * A set of utility functions ported from io, ioutil & hugo to be afero aware - +* Wrapper for go 1.16 filesystem abstraction `io/fs.FS` # Using Afero diff --git a/iofs.go b/iofs.go new file mode 100644 index 00000000..c8034553 --- /dev/null +++ b/iofs.go @@ -0,0 +1,288 @@ +// +build go1.16 + +package afero + +import ( + "io" + "io/fs" + "os" + "path" + "time" +) + +// IOFS adopts afero.Fs to stdlib io/fs.FS +type IOFS struct { + Fs +} + +func NewIOFS(fs Fs) IOFS { + return IOFS{Fs: fs} +} + +var ( + _ fs.FS = IOFS{} + _ fs.GlobFS = IOFS{} + _ fs.ReadDirFS = IOFS{} + _ fs.ReadFileFS = IOFS{} + _ fs.StatFS = IOFS{} + _ fs.SubFS = IOFS{} +) + +func (iofs IOFS) Open(name string) (fs.File, error) { + const op = "open" + + // by convention for fs.FS implementations we should perform this check + if !fs.ValidPath(name) { + return nil, iofs.wrapError(op, name, fs.ErrInvalid) + } + + file, err := iofs.Fs.Open(name) + if err != nil { + return nil, iofs.wrapError(op, name, err) + } + + // file should implement fs.ReadDirFile + if _, ok := file.(fs.ReadDirFile); !ok { + file = readDirFile{file} + } + + return file, nil +} + +func (iofs IOFS) Glob(pattern string) ([]string, error) { + const op = "glob" + + // afero.Glob does not perform this check but it's required for implementations + if _, err := path.Match(pattern, ""); err != nil { + return nil, iofs.wrapError(op, pattern, err) + } + + items, err := Glob(iofs.Fs, pattern) + if err != nil { + return nil, iofs.wrapError(op, pattern, err) + } + + return items, nil +} + +func (iofs IOFS) ReadDir(name string) ([]fs.DirEntry, error) { + items, err := ReadDir(iofs.Fs, name) + if err != nil { + return nil, iofs.wrapError("readdir", name, err) + } + + ret := make([]fs.DirEntry, len(items)) + for i := range items { + ret[i] = dirEntry{items[i]} + } + + return ret, nil +} + +func (iofs IOFS) ReadFile(name string) ([]byte, error) { + const op = "readfile" + + if !fs.ValidPath(name) { + return nil, iofs.wrapError(op, name, fs.ErrInvalid) + } + + bytes, err := ReadFile(iofs.Fs, name) + if err != nil { + return nil, iofs.wrapError(op, name, err) + } + + return bytes, nil +} + +func (iofs IOFS) Sub(dir string) (fs.FS, error) { return IOFS{NewBasePathFs(iofs.Fs, dir)}, nil } + +func (IOFS) wrapError(op, path string, err error) error { + if _, ok := err.(*fs.PathError); ok { + return err // don't need to wrap again + } + + return &fs.PathError{ + Op: op, + Path: path, + Err: err, + } +} + +// dirEntry provides adapter from os.FileInfo to fs.DirEntry +type dirEntry struct { + fs.FileInfo +} + +var _ fs.DirEntry = dirEntry{} + +func (d dirEntry) Type() fs.FileMode { return d.FileInfo.Mode().Type() } + +func (d dirEntry) Info() (fs.FileInfo, error) { return d.FileInfo, nil } + +// readDirFile provides adapter from afero.File to fs.ReadDirFile needed for correct Open +type readDirFile struct { + File +} + +var _ fs.ReadDirFile = readDirFile{} + +func (r readDirFile) ReadDir(n int) ([]fs.DirEntry, error) { + items, err := r.File.Readdir(n) + if err != nil { + return nil, err + } + + ret := make([]fs.DirEntry, len(items)) + for i := range items { + ret[i] = dirEntry{items[i]} + } + + return ret, nil +} + +// FromIOFS adopts io/fs.FS to use it as afero.Fs +// Note that io/fs.FS is read-only so all mutating methods will return fs.PathError with fs.ErrPermission +// To store modifications you may use afero.CopyOnWriteFs +type FromIOFS struct { + fs.FS +} + +var _ Fs = FromIOFS{} + +func (f FromIOFS) Create(name string) (File, error) { return nil, notImplemented("create", name) } + +func (f FromIOFS) Mkdir(name string, perm os.FileMode) error { return notImplemented("mkdir", name) } + +func (f FromIOFS) MkdirAll(path string, perm os.FileMode) error { + return notImplemented("mkdirall", path) +} + +func (f FromIOFS) Open(name string) (File, error) { + file, err := f.FS.Open(name) + if err != nil { + return nil, err + } + + return fromIOFSFile{File: file, name: name}, nil +} + +func (f FromIOFS) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return f.Open(name) +} + +func (f FromIOFS) Remove(name string) error { + return notImplemented("remove", name) +} + +func (f FromIOFS) RemoveAll(path string) error { + return notImplemented("removeall", path) +} + +func (f FromIOFS) Rename(oldname, newname string) error { + return notImplemented("rename", oldname) +} + +func (f FromIOFS) Stat(name string) (os.FileInfo, error) { return fs.Stat(f.FS, name) } + +func (f FromIOFS) Name() string { return "fromiofs" } + +func (f FromIOFS) Chmod(name string, mode os.FileMode) error { + return notImplemented("chmod", name) +} + +func (f FromIOFS) Chown(name string, uid, gid int) error { + return notImplemented("chown", name) +} + +func (f FromIOFS) Chtimes(name string, atime time.Time, mtime time.Time) error { + return notImplemented("chtimes", name) +} + +type fromIOFSFile struct { + fs.File + name string +} + +func (f fromIOFSFile) ReadAt(p []byte, off int64) (n int, err error) { + readerAt, ok := f.File.(io.ReaderAt) + if !ok { + return -1, notImplemented("readat", f.name) + } + + return readerAt.ReadAt(p, off) +} + +func (f fromIOFSFile) Seek(offset int64, whence int) (int64, error) { + seeker, ok := f.File.(io.Seeker) + if !ok { + return -1, notImplemented("seek", f.name) + } + + return seeker.Seek(offset, whence) +} + +func (f fromIOFSFile) Write(p []byte) (n int, err error) { + return -1, notImplemented("write", f.name) +} + +func (f fromIOFSFile) WriteAt(p []byte, off int64) (n int, err error) { + return -1, notImplemented("writeat", f.name) +} + +func (f fromIOFSFile) Name() string { return f.name } + +func (f fromIOFSFile) Readdir(count int) ([]os.FileInfo, error) { + rdfile, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, notImplemented("readdir", f.name) + } + + entries, err := rdfile.ReadDir(count) + if err != nil { + return nil, err + } + + ret := make([]os.FileInfo, len(entries)) + for i := range entries { + ret[i], err = entries[i].Info() + + if err != nil { + return nil, err + } + } + + return ret, nil +} + +func (f fromIOFSFile) Readdirnames(n int) ([]string, error) { + rdfile, ok := f.File.(fs.ReadDirFile) + if !ok { + return nil, notImplemented("readdir", f.name) + } + + entries, err := rdfile.ReadDir(n) + if err != nil { + return nil, err + } + + ret := make([]string, len(entries)) + for i := range entries { + ret[i] = entries[i].Name() + } + + return ret, nil +} + +func (f fromIOFSFile) Sync() error { return nil } + +func (f fromIOFSFile) Truncate(size int64) error { + return notImplemented("truncate", f.name) +} + +func (f fromIOFSFile) WriteString(s string) (ret int, err error) { + return -1, notImplemented("writestring", f.name) +} + +func notImplemented(op, path string) error { + return &fs.PathError{Op: op, Path: path, Err: fs.ErrPermission} +} diff --git a/iofs_test.go b/iofs_test.go new file mode 100644 index 00000000..1d310e54 --- /dev/null +++ b/iofs_test.go @@ -0,0 +1,412 @@ +// +build go1.16 + +package afero + +import ( + "bytes" + "errors" + "io" + "io/fs" + "os" + "testing" + "testing/fstest" + "time" +) + +func TestIOFS(t *testing.T) { + t.Parallel() + + t.Run("use MemMapFs", func(t *testing.T) { + mmfs := NewMemMapFs() + + err := mmfs.MkdirAll("dir1/dir2", os.ModePerm) + if err != nil { + t.Fatal("MkdirAll failed:", err) + } + + f, err := mmfs.OpenFile("dir1/dir2/test.txt", os.O_RDWR|os.O_CREATE, os.ModePerm) + if err != nil { + t.Fatal("OpenFile (O_CREATE) failed:", err) + } + + f.Close() + + if err := fstest.TestFS(NewIOFS(mmfs), "dir1/dir2/test.txt"); err != nil { + t.Error(err) + } + }) + + t.Run("use OsFs", func(t *testing.T) { + osfs := NewBasePathFs(NewOsFs(), t.TempDir()) + + err := osfs.MkdirAll("dir1/dir2", os.ModePerm) + if err != nil { + t.Fatal("MkdirAll failed:", err) + } + + f, err := osfs.OpenFile("dir1/dir2/test.txt", os.O_RDWR|os.O_CREATE, os.ModePerm) + if err != nil { + t.Fatal("OpenFile (O_CREATE) failed:", err) + } + + f.Close() + + if err := fstest.TestFS(NewIOFS(osfs), "dir1/dir2/test.txt"); err != nil { + t.Error(err) + } + }) +} + +func TestFromIOFS(t *testing.T) { + t.Parallel() + + fsys := fstest.MapFS{ + "test.txt": { + Data: []byte("File in root"), + Mode: fs.ModePerm, + ModTime: time.Now(), + }, + "dir1": { + Mode: fs.ModeDir | fs.ModePerm, + ModTime: time.Now(), + }, + "dir1/dir2": { + Mode: fs.ModeDir | fs.ModePerm, + ModTime: time.Now(), + }, + "dir1/dir2/hello.txt": { + Data: []byte("Hello world"), + Mode: fs.ModePerm, + ModTime: time.Now(), + }, + } + + fromIOFS := FromIOFS{fsys} + + t.Run("Create", func(t *testing.T) { + _, err := fromIOFS.Create("test") + assertPermissionError(t, err) + }) + + t.Run("Mkdir", func(t *testing.T) { + err := fromIOFS.Mkdir("test", 0) + assertPermissionError(t, err) + }) + + t.Run("MkdirAll", func(t *testing.T) { + err := fromIOFS.Mkdir("test", 0) + assertPermissionError(t, err) + }) + + t.Run("Open", func(t *testing.T) { + t.Run("non existing file", func(t *testing.T) { + _, err := fromIOFS.Open("nonexisting") + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("Expected error to be fs.ErrNotExist, got %[1]T (%[1]v)", err) + } + }) + + t.Run("directory", func(t *testing.T) { + dirFile, err := fromIOFS.Open("dir1") + if err != nil { + t.Errorf("dir1 open failed: %v", err) + return + } + + defer dirFile.Close() + + dirStat, err := dirFile.Stat() + if err != nil { + t.Errorf("dir1 stat failed: %v", err) + return + } + + if !dirStat.IsDir() { + t.Errorf("dir1 stat told that it is not a directory") + return + } + }) + + t.Run("simple file", func(t *testing.T) { + file, err := fromIOFS.Open("test.txt") + if err != nil { + t.Errorf("test.txt open failed: %v", err) + return + } + + defer file.Close() + + fileStat, err := file.Stat() + if err != nil { + t.Errorf("test.txt stat failed: %v", err) + return + } + + if fileStat.IsDir() { + t.Errorf("test.txt stat told that it is a directory") + return + } + }) + }) + + t.Run("Remove", func(t *testing.T) { + err := fromIOFS.Remove("test") + assertPermissionError(t, err) + }) + + t.Run("Rename", func(t *testing.T) { + err := fromIOFS.Rename("test", "test2") + assertPermissionError(t, err) + }) + + t.Run("Stat", func(t *testing.T) { + t.Run("non existing file", func(t *testing.T) { + _, err := fromIOFS.Stat("nonexisting") + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("Expected error to be fs.ErrNotExist, got %[1]T (%[1]v)", err) + } + }) + + t.Run("directory", func(t *testing.T) { + stat, err := fromIOFS.Stat("dir1/dir2") + if err != nil { + t.Errorf("dir1/dir2 stat failed: %v", err) + return + } + + if !stat.IsDir() { + t.Errorf("dir1/dir2 stat told that it is not a directory") + return + } + }) + + t.Run("file", func(t *testing.T) { + stat, err := fromIOFS.Stat("dir1/dir2/hello.txt") + if err != nil { + t.Errorf("dir1/dir2 stat failed: %v", err) + return + } + + if stat.IsDir() { + t.Errorf("dir1/dir2/hello.txt stat told that it is a directory") + return + } + + if lenFile := len(fsys["dir1/dir2/hello.txt"].Data); int64(lenFile) != stat.Size() { + t.Errorf("dir1/dir2/hello.txt stat told invalid size: expected %d, got %d", lenFile, stat.Size()) + return + } + }) + }) + + t.Run("Chmod", func(t *testing.T) { + err := fromIOFS.Chmod("test", os.ModePerm) + assertPermissionError(t, err) + }) + + t.Run("Chown", func(t *testing.T) { + err := fromIOFS.Chown("test", 0, 0) + assertPermissionError(t, err) + }) + + t.Run("Chtimes", func(t *testing.T) { + err := fromIOFS.Chtimes("test", time.Now(), time.Now()) + assertPermissionError(t, err) + }) +} + +func TestFromIOFS_File(t *testing.T) { + t.Parallel() + + fsys := fstest.MapFS{ + "test.txt": { + Data: []byte("File in root"), + Mode: fs.ModePerm, + ModTime: time.Now(), + }, + "dir1": { + Mode: fs.ModeDir | fs.ModePerm, + ModTime: time.Now(), + }, + "dir2": { + Mode: fs.ModeDir | fs.ModePerm, + ModTime: time.Now(), + }, + } + + fromIOFS := FromIOFS{fsys} + + file, err := fromIOFS.Open("test.txt") + if err != nil { + t.Errorf("test.txt open failed: %v", err) + return + } + + defer file.Close() + + fileStat, err := file.Stat() + if err != nil { + t.Errorf("test.txt stat failed: %v", err) + return + } + + if fileStat.IsDir() { + t.Errorf("test.txt stat told that it is a directory") + return + } + + t.Run("ReadAt", func(t *testing.T) { + // MapFS files implements io.ReaderAt + b := make([]byte, 2) + _, err := file.ReadAt(b, 2) + + if err != nil { + t.Errorf("ReadAt failed: %v", err) + return + } + + if expectedData := fsys["test.txt"].Data[2:4]; !bytes.Equal(b, expectedData) { + t.Errorf("Unexpected content read: %s, expected %s", b, expectedData) + } + }) + + t.Run("Seek", func(t *testing.T) { + n, err := file.Seek(2, io.SeekStart) + if err != nil { + t.Errorf("Seek failed: %v", err) + return + } + + if n != 2 { + t.Errorf("Seek returned unexpected value: %d, expected 2", n) + } + }) + + t.Run("Write", func(t *testing.T) { + _, err := file.Write(nil) + assertPermissionError(t, err) + }) + + t.Run("WriteAt", func(t *testing.T) { + _, err := file.WriteAt(nil, 0) + assertPermissionError(t, err) + }) + + t.Run("Name", func(t *testing.T) { + if name := file.Name(); name != "test.txt" { + t.Errorf("expected file.Name() == test.txt, got %s", name) + } + }) + + t.Run("Readdir", func(t *testing.T) { + t.Run("not directory", func(t *testing.T) { + _, err := file.Readdir(-1) + assertPermissionError(t, err) + }) + + t.Run("root directory", func(t *testing.T) { + root, err := fromIOFS.Open(".") + if err != nil { + t.Errorf("root open failed: %v", err) + return + } + + defer root.Close() + + items, err := root.Readdir(-1) + if err != nil { + t.Errorf("Readdir error: %v", err) + return + } + + var expectedItems = []struct { + Name string + IsDir bool + Size int64 + }{ + {Name: "dir1", IsDir: true, Size: 0}, + {Name: "dir2", IsDir: true, Size: 0}, + {Name: "test.txt", IsDir: false, Size: int64(len(fsys["test.txt"].Data))}, + } + + if len(expectedItems) != len(items) { + t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items)) + return + } + + for i, item := range items { + if item.Name() != expectedItems[i].Name { + t.Errorf("Item %d: expected name %s, got %s", i, expectedItems[i].Name, item.Name()) + } + + if item.IsDir() != expectedItems[i].IsDir { + t.Errorf("Item %d: expected IsDir %t, got %t", i, expectedItems[i].IsDir, item.IsDir()) + } + + if item.Size() != expectedItems[i].Size { + t.Errorf("Item %d: expected IsDir %d, got %d", i, expectedItems[i].Size, item.Size()) + } + } + }) + }) + + t.Run("Readdirnames", func(t *testing.T) { + t.Run("not directory", func(t *testing.T) { + _, err := file.Readdirnames(-1) + assertPermissionError(t, err) + }) + + t.Run("root directory", func(t *testing.T) { + root, err := fromIOFS.Open(".") + if err != nil { + t.Errorf("root open failed: %v", err) + return + } + + defer root.Close() + + items, err := root.Readdirnames(-1) + if err != nil { + t.Errorf("Readdirnames error: %v", err) + return + } + + var expectedItems = []string{"dir1", "dir2", "test.txt"} + + if len(expectedItems) != len(items) { + t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items)) + return + } + + for i, item := range items { + if item != expectedItems[i] { + t.Errorf("Item %d: expected name %s, got %s", i, expectedItems[i], item) + } + } + }) + }) + + t.Run("Truncate", func(t *testing.T) { + err := file.Truncate(1) + assertPermissionError(t, err) + }) + + t.Run("WriteString", func(t *testing.T) { + _, err := file.WriteString("a") + assertPermissionError(t, err) + }) +} + +func assertPermissionError(t *testing.T, err error) { + t.Helper() + + var perr *fs.PathError + if !errors.As(err, &perr) { + t.Errorf("Expected *fs.PathError, got %[1]T (%[1]v)", err) + return + } + + if perr.Err != fs.ErrPermission { + t.Errorf("Expected (*fs.PathError).Err == fs.ErrPermisson, got %[1]T (%[1]v)", err) + } +}